diff --git a/csrc/apis/mega_moe.hpp b/csrc/apis/mega_moe.hpp index c5d96cf4d..2dc09ebee 100644 --- a/csrc/apis/mega_moe.hpp +++ b/csrc/apis/mega_moe.hpp @@ -318,7 +318,8 @@ static void fp8_fp4_mega_moe( const bool& fast_math, const float& activation_alpha, const float& activation_beta, - const std::optional& num_tokens_for_block_m + const std::optional& num_tokens_for_block_m, + const std::optional& shared_expert_weights_opt ) { const auto [l1_weights, l1_weights_sf] = l1_weights_tuple; const auto [l2_weights, l2_weights_sf] = l2_weights_tuple; @@ -398,6 +399,20 @@ static void fp8_fp4_mega_moe( DG_HOST_ASSERT(cumulative_local_expert_recv_stats->is_contiguous()); } + // Check shared expert weights, e.g. Qwen's `sigmoid(shared_expert_gate(x))`, applied where routed tokens + // apply their top-k weight + const float* shared_expert_weights = nullptr; + if (shared_expert_weights_opt.has_value()) { + const auto& weights = shared_expert_weights_opt.value(); + DG_HOST_ASSERT(weights.scalar_type() == torch::headeronly::ScalarType::Float and weights.is_contiguous()); + DG_HOST_ASSERT(weights.dim() == 1 and weights.size(0) == num_tokens); + DG_HOST_ASSERT(weights.is_cuda() and weights.get_device_index() == y.get_device_index()); + DG_HOST_ASSERT(num_shared_experts > 0); + // NOTES: only the SM100 SwiGLU kernel implements shared expert weights + DG_HOST_ASSERT(arch_major == 10 and activation != "situ"); + shared_expert_weights = weights.const_data_ptr(); + } + // Check buffer bytes const auto num_ranks = static_cast(sym_buffer_ptrs.size()); const auto num_experts_ = num_experts_per_rank * num_ranks; @@ -445,7 +460,8 @@ static void fp8_fp4_mega_moe( activation_alpha, activation_beta, fast_math); } else { launch(sm100_fp8_fp4_mega_moe, - activation_clamp, activation == "swiglustep", activation_alpha, activation_beta, fast_math); + activation_clamp, activation == "swiglustep", activation_alpha, activation_beta, fast_math, + shared_expert_weights); } } else if (arch_major == 9) { DG_HOST_ASSERT(activation != "situ"); @@ -694,7 +710,8 @@ static void bf16_mega_moe( const float& activation_alpha, const float& activation_beta, const std::optional& post_expert_norm_gamma_opt, - const float& post_expert_norm_eps + const float& post_expert_norm_eps, + const std::optional& shared_expert_weights_opt ) { // Config checks const auto num_tokens = static_cast(y.size(0)); @@ -780,6 +797,18 @@ static void bf16_mega_moe( DG_HOST_ASSERT(torch_compat::nbytes(sym_buffer) >= static_cast(layout_info.num_bytes)); DG_HOST_ASSERT(num_experts == num_experts_); + // Check shared expert weights, e.g. Qwen's `sigmoid(shared_expert_gate(x))`, applied where routed tokens + // apply their top-k weight + const float* shared_expert_weights = nullptr; + if (shared_expert_weights_opt.has_value()) { + const auto& weights = shared_expert_weights_opt.value(); + DG_HOST_ASSERT(weights.scalar_type() == torch::headeronly::ScalarType::Float and weights.is_contiguous()); + DG_HOST_ASSERT(weights.dim() == 1 and weights.size(0) == num_tokens); + DG_HOST_ASSERT(weights.is_cuda() and weights.get_device_index() == y.get_device_index()); + DG_HOST_ASSERT(num_shared_experts > 0); + shared_expert_weights = weights.const_data_ptr(); + } + const void* post_expert_norm_gamma_ptr = nullptr; if (post_expert_norm_gamma_opt.has_value()) { const auto& gamma = post_expert_norm_gamma_opt.value(); @@ -814,7 +843,8 @@ static void bf16_mega_moe( hidden, intermediate_hidden, activation_clamp, activation == "swiglustep", activation_alpha, activation_beta, fast_math, - post_expert_norm_gamma_ptr, post_expert_norm_eps); + post_expert_norm_gamma_ptr, post_expert_norm_eps, + shared_expert_weights); } else { DG_HOST_UNREACHABLE("Unsupported architecture"); } @@ -884,7 +914,8 @@ static void fp8_fp4_mega_moe( const bool& fast_math, const double& activation_alpha, const double& activation_beta, - const std::optional& num_tokens_for_block_m) { + const std::optional& num_tokens_for_block_m, + const std::optional& shared_expert_weights_opt) { mega::fp8_fp4_mega_moe( y, {l1_weights_tuple, l1_weights_tuple_sf}, {l2_weights_tuple, l2_weights_tuple_sf}, make_optional_tensor_pair(shared_l1_weights_tuple_opt, shared_l1_weights_tuple_opt_sf), @@ -893,7 +924,8 @@ static void fp8_fp4_mega_moe( mega::checked_int(rank_idx), mega::checked_int(num_max_tokens_per_rank), mega::checked_int(num_experts), mega::checked_int(num_topk), checked_recipe(recipe), activation, activation_clamp_opt, fast_math, - static_cast(activation_alpha), static_cast(activation_beta), num_tokens_for_block_m); + static_cast(activation_alpha), static_cast(activation_beta), num_tokens_for_block_m, + shared_expert_weights_opt); } template @@ -916,7 +948,8 @@ static void bf16_mega_moe( const double& activation_alpha, const double& activation_beta, const std::optional& post_expert_norm_gamma_opt, - const double& post_expert_norm_eps) { + const double& post_expert_norm_eps, + const std::optional& shared_expert_weights_opt) { mega::bf16_mega_moe( y, l1_weights, l2_weights, shared_l1_weights_opt, shared_l2_weights_opt, cumulative_local_expert_recv_stats, sym_buffer, sym_buffer_ptrs, @@ -924,7 +957,8 @@ static void bf16_mega_moe( mega::checked_int(num_experts), mega::checked_int(num_topk), activation, activation_clamp_opt, fast_math, static_cast(activation_alpha), static_cast(activation_beta), - post_expert_norm_gamma_opt, static_cast(post_expert_norm_eps)); + post_expert_norm_gamma_opt, static_cast(post_expert_norm_eps), + shared_expert_weights_opt); } template @@ -989,9 +1023,9 @@ STABLE_TORCH_LIBRARY_FRAGMENT(deep_gemm, m) { m.def("get_symm_buffer_size_for_mega_moe(int num_ranks, int num_experts, int num_max_tokens_per_rank, int num_topk, int hidden, int intermediate_hidden, str mma_type, str activation, int num_shared_experts) -> (int, int[])"); m.def("_slice_symm_buffer_for_mega_moe(Tensor(a) buffer, int[] layout_info) -> (Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a))"); - m.def("fp8_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_tuple_opt, Tensor? shared_l1_weights_tuple_opt_sf, Tensor? shared_l2_weights_tuple_opt, Tensor? shared_l2_weights_tuple_opt_sf, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, int? num_tokens_for_block_m) -> ()"); - m.def("bf16_mega_moe(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()"); - m.def("bf16_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()"); + m.def("fp8_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_tuple_opt, Tensor? shared_l1_weights_tuple_opt_sf, Tensor? shared_l2_weights_tuple_opt, Tensor? shared_l2_weights_tuple_opt_sf, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, int? num_tokens_for_block_m, Tensor? shared_expert_weights_opt=None) -> ()"); + m.def("bf16_mega_moe(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps, Tensor? shared_expert_weights_opt=None) -> ()"); + m.def("bf16_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps, Tensor? shared_expert_weights_opt=None) -> ()"); m.def("fp4_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor? x_bf16_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, Tensor? l1_alphas_opt, Tensor? l2_alphas_opt, Tensor? a2_scales_opt, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()"); m.def("fp4_fp4_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor? x_bf16_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, Tensor? l1_alphas_opt, Tensor? l2_alphas_opt, Tensor? a2_scales_opt, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()"); } diff --git a/csrc/jit_kernels/impls/sm100_bf16_mega_moe.hpp b/csrc/jit_kernels/impls/sm100_bf16_mega_moe.hpp index 1550c7a7f..0b634ff1d 100644 --- a/csrc/jit_kernels/impls/sm100_bf16_mega_moe.hpp +++ b/csrc/jit_kernels/impls/sm100_bf16_mega_moe.hpp @@ -69,7 +69,8 @@ static void sm100_bf16_mega_moe( const float& activation_alpha, const float& activation_beta, const bool& fast_math, - const void* post_expert_norm_gamma, const float& post_expert_norm_eps + const void* post_expert_norm_gamma, const float& post_expert_norm_eps, + const float* shared_expert_weights ) { const auto num_ranks = static_cast(sym_buffer_ptrs.size()); const auto num_experts = num_experts_per_rank * num_ranks; @@ -166,6 +167,7 @@ static void __instantiate_kernel() {{ {}, {}, {}, + {}, {} >); }}; @@ -186,6 +188,7 @@ static void __instantiate_kernel() {{ to_string(activation_beta), fast_math ? "true" : "false", post_expert_norm_gamma != nullptr ? "true" : "false", + shared_expert_weights != nullptr ? "true" : "false", use_trtllm_weights ? "true" : "false")); // Launch @@ -199,6 +202,7 @@ static void __instantiate_kernel() {{ y.mutable_data_ptr(), cumulative_local_expert_recv_stats_ptr, num_tokens, + shared_expert_weights, post_expert_norm_gamma, post_expert_norm_eps, layout::SymBuffer<>(sym_buffer_ptrs, rank_idx), tensor_map_l1_acts, diff --git a/csrc/jit_kernels/impls/sm100_fp8_fp4_mega_moe.hpp b/csrc/jit_kernels/impls/sm100_fp8_fp4_mega_moe.hpp index 3d25cbbc3..5fa15b39d 100644 --- a/csrc/jit_kernels/impls/sm100_fp8_fp4_mega_moe.hpp +++ b/csrc/jit_kernels/impls/sm100_fp8_fp4_mega_moe.hpp @@ -36,7 +36,8 @@ static void sm100_fp8_fp4_mega_moe( const bool& clamp_after_silu, const float& activation_alpha, const float& activation_beta, - const bool& fast_math + const bool& fast_math, + const float* shared_expert_weights ) { const auto num_ranks = static_cast(sym_buffer_ptrs.size()); const auto num_experts = num_experts_per_rank * num_ranks; @@ -175,6 +176,7 @@ static void __instantiate_kernel() {{ {}, {}, {}, + {}, {} >); }}; @@ -196,6 +198,7 @@ static void __instantiate_kernel() {{ to_string(activation_alpha), to_string(activation_beta), fast_math ? "true" : "false", + shared_expert_weights != nullptr ? "true" : "false", to_string(l1_weights.scalar_type()))); // Launch @@ -209,6 +212,7 @@ static void __instantiate_kernel() {{ y.mutable_data_ptr(), cumulative_local_expert_recv_stats_ptr, num_tokens, + shared_expert_weights, layout::SymBuffer<>(sym_buffer_ptrs, rank_idx), tensor_map_l1_acts, tensor_map_l1_acts_sf, diff --git a/deep_gemm/_C.py b/deep_gemm/_C.py index 5a6235291..576e276c0 100644 --- a/deep_gemm/_C.py +++ b/deep_gemm/_C.py @@ -514,6 +514,7 @@ def fp8_fp4_mega_moe( activation_alpha, activation_beta, num_tokens_for_block_m=None, + shared_expert_weights_opt=None, ): return _torch_ops.fp8_fp4_mega_moe( y, @@ -539,6 +540,7 @@ def fp8_fp4_mega_moe( activation_alpha, activation_beta, num_tokens_for_block_m, + shared_expert_weights_opt, ) @@ -562,6 +564,7 @@ def bf16_mega_moe( activation_beta, post_expert_norm_gamma_opt=None, post_expert_norm_eps=0.0, + shared_expert_weights_opt=None, ): return _torch_ops.bf16_mega_moe( y, @@ -583,6 +586,7 @@ def bf16_mega_moe( activation_beta, post_expert_norm_gamma_opt, post_expert_norm_eps, + shared_expert_weights_opt, ) @@ -606,6 +610,7 @@ def bf16_mega_moe_trtllm( activation_beta, post_expert_norm_gamma_opt=None, post_expert_norm_eps=0.0, + shared_expert_weights_opt=None, ): return _torch_ops.bf16_mega_moe_trtllm( y, @@ -627,6 +632,7 @@ def bf16_mega_moe_trtllm( activation_beta, post_expert_norm_gamma_opt, post_expert_norm_eps, + shared_expert_weights_opt, ) diff --git a/deep_gemm/include/deep_gemm/impls/sm100_bf16_mega_moe.cuh b/deep_gemm/include/deep_gemm/impls/sm100_bf16_mega_moe.cuh index 7d74fcb92..8d70f6a3b 100644 --- a/deep_gemm/include/deep_gemm/impls/sm100_bf16_mega_moe.cuh +++ b/deep_gemm/include/deep_gemm/impls/sm100_bf16_mega_moe.cuh @@ -37,6 +37,7 @@ template < float kActivationBeta, bool kFastMath, bool kWithPostExpertNorm, + bool kWithSharedExpertWeights, bool kUseTrtllmWeights = false, bool kHasShared = (kNumSharedExperts > 0), uint32_t L1_SHAPE_N = kIntermediateHidden * 2, @@ -58,6 +59,7 @@ CUTLASS_GLOBAL __launch_bounds__(kNumThreads, 1) void sm100_bf16_mega_moe_impl(void* y, int* cumulative_local_expert_recv_stats, const uint32_t num_tokens, + const float* shared_expert_weights, const float* post_expert_norm_gamma, const float post_expert_norm_eps, const __grid_constant__ layout::SymBuffer sym_buffer, @@ -141,6 +143,7 @@ sm100_bf16_mega_moe_impl(void* y, DG_STATIC_ASSERT(not kUseTrtllmWeights or (not kHasShared and BLOCK_K == 64 and LOAD_BLOCK_N % 32 == 0), "TRT-LLM BF16 weights require no shared experts, 64-element K blocks and 32-row aligned tiles"); DG_STATIC_ASSERT(BLOCK_M % 16 == 0, "Invalid block M"); + DG_STATIC_ASSERT(not kWithSharedExpertWeights or kHasShared, "Shared expert weights require fused shared experts"); DG_STATIC_ASSERT(BLOCK_N == LAYOUT_AD_M, "Invalid block N"); // Swizzle configs @@ -943,6 +946,15 @@ sm100_bf16_mega_moe_impl(void* y, .get_data_buffer(ring_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx) .template get_base_ptr(); } + // Optional per-token shared-expert weight (e.g. Qwen's `sigmoid(shared_expert_gate(x))`), + // applied where routed tokens apply their top-k weight. Shared rows are local token indices. + if constexpr (kWithSharedExpertWeights) { + if (task_info.is_shared() and (j * ATOM_M) % 32 == 0 and + (WG_BLOCK_M % 32 == 0 or j * ATOM_M + lane_idx < WG_BLOCK_M)) { + const uint32_t token_idx = pool_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx; + stored_cached_weight = token_idx < num_tokens ? __ldg(shared_expert_weights + token_idx) : 0.0f; + } + } // Load weights from register cache const float2 weights = { diff --git a/deep_gemm/include/deep_gemm/impls/sm100_fp8_fp4_mega_moe.cuh b/deep_gemm/include/deep_gemm/impls/sm100_fp8_fp4_mega_moe.cuh index 05d850271..d170b6c93 100644 --- a/deep_gemm/include/deep_gemm/impls/sm100_fp8_fp4_mega_moe.cuh +++ b/deep_gemm/include/deep_gemm/impls/sm100_fp8_fp4_mega_moe.cuh @@ -38,6 +38,7 @@ template < float kActivationAlpha, float kActivationBeta, bool kFastMath, + bool kWithSharedExpertWeights, typename weight_dtype_t, bool kHasShared = (kNumSharedExperts > 0), uint32_t L1_SHAPE_N = kIntermediateHidden * 2, @@ -60,6 +61,7 @@ CUTLASS_GLOBAL __launch_bounds__(kNumThreads, 1) void sm100_fp8_fp4_mega_moe_impl(void* y, int* cumulative_local_expert_recv_stats, const uint32_t num_tokens, + const float* shared_expert_weights, const __grid_constant__ layout::SymBuffer sym_buffer, const __grid_constant__ cute::TmaDescriptor tensor_map_l1_acts, const __grid_constant__ cute::TmaDescriptor tensor_map_l1_acts_sf, @@ -94,6 +96,7 @@ sm100_fp8_fp4_mega_moe_impl(void* y, DG_STATIC_ASSERT(kNumNonEpilogueThreads == 128, "Invalid number of MMA non-epilogue threads"); DG_STATIC_ASSERT(kNumEpilogueThreads % 128 == 0, "Invalid number of MMA epilogue and combine threads"); DG_STATIC_ASSERT(kNumExperts % kNumRanks == 0, "Invalid number of experts or ranks"); + DG_STATIC_ASSERT(not kWithSharedExpertWeights or kHasShared, "Shared expert weights require fused shared experts"); // Thread indices const bool is_leader_cta = cute::block_rank_in_cluster() == 0; @@ -1101,6 +1104,15 @@ sm100_fp8_fp4_mega_moe_impl(void* y, .get_data_buffer(ring_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx) .template get_base_ptr(); } + // Optional per-token shared-expert weight (e.g. Qwen's `sigmoid(shared_expert_gate(x))`), + // applied where routed tokens apply their top-k weight. Shared rows are local token indices. + if constexpr (kWithSharedExpertWeights) { + if (task_info.is_shared() and (j * ATOM_M) % 32 == 0 and + (WG_BLOCK_M % 32 == 0 or j * ATOM_M + lane_idx < WG_BLOCK_M)) { + const uint32_t token_idx = pool_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx; + stored_cached_weight = token_idx < num_tokens ? __ldg(shared_expert_weights + token_idx) : 0.0f; + } + } // Load weights from register cache const float2 weights = { diff --git a/deep_gemm/mega/__init__.py b/deep_gemm/mega/__init__.py index d49d6eb36..f4c8bce24 100644 --- a/deep_gemm/mega/__init__.py +++ b/deep_gemm/mega/__init__.py @@ -292,8 +292,14 @@ def fp8_fp4_mega_moe(y: torch.Tensor, fast_math: bool = True, activation_alpha: float = 1.0, activation_beta: float = 0.0, - num_tokens_for_block_m: Optional[int] = None): + num_tokens_for_block_m: Optional[int] = None, + shared_expert_weights: Optional[torch.Tensor] = None): # SM90: `num_tokens_for_block_m` (default: this rank's tokens) picks BLOCK_M; pass one value on all ranks + # `shared_expert_weights` is an optional FP32 `(num_tokens, )` per-token weight of the fused shared + # experts' output, e.g. Qwen's `sigmoid(shared_expert_gate(x))` (the caller computes it). The kernel + # applies it after the activation, the same place routed experts apply their top-k weights. + # `None` keeps the unweighted sum. Under CUDA graphs, it must have a stable address, like `y`. + # SM100 SwiGLU only (not SM90 or SiTU). _validate_activation(activation) recipe = recipe or ((1, 1, 128) if torch.cuda.get_device_capability(y.device)[0] == 9 else (1, 1, 32)) _C.fp8_fp4_mega_moe( @@ -309,7 +315,8 @@ def fp8_fp4_mega_moe(y: torch.Tensor, activation, activation_clamp, fast_math, activation_alpha, activation_beta, - num_tokens_for_block_m + num_tokens_for_block_m, + shared_expert_weights ) def fp4_fp4_mega_moe(y: torch.Tensor, @@ -399,8 +406,12 @@ def bf16_mega_moe(y: torch.Tensor, activation_beta: float = 0.0, post_expert_norm_gamma: Optional[torch.Tensor] = None, post_expert_norm_eps: float = 0.0, - weight_layout: str = "native"): - """``weight_layout="trtllm"`` takes TRT-LLM's BF16 MoE weights as is (no shared experts).""" + weight_layout: str = "native", + shared_expert_weights: Optional[torch.Tensor] = None): + """``weight_layout="trtllm"`` takes TRT-LLM's BF16 MoE weights as is (no shared experts). + + ``shared_expert_weights`` is as in :func:`fp8_fp4_mega_moe`. + """ assert activation in ('swiglu', 'swiglustep'), \ f'BF16 Mega MoE supports `swiglu` or `swiglustep`, got `{activation}`' if weight_layout not in ("native", "trtllm"): @@ -422,5 +433,6 @@ def bf16_mega_moe(y: torch.Tensor, activation, activation_clamp, fast_math, activation_alpha, activation_beta, - post_expert_norm_gamma, post_expert_norm_eps + post_expert_norm_gamma, post_expert_norm_eps, + shared_expert_weights ) diff --git a/tests/test_mega_moe.py b/tests/test_mega_moe.py index e2f630976..87a42b48f 100644 --- a/tests/test_mega_moe.py +++ b/tests/test_mega_moe.py @@ -131,7 +131,7 @@ def _hi_expert(x: torch.Tensor, l1_weights: Tuple, l2_weights: Tuple, weights: t def _moe_reference(expert, xs: Tuple, topk_idx: torch.Tensor, topk_weights: torch.Tensor, l1_weights: Tuple, l2_weights: Tuple, shared_weights: Optional[Tuple], - dtype: torch.dtype) -> torch.Tensor: + shared_expert_weights: Optional[torch.Tensor], dtype: torch.dtype) -> torch.Tensor: # Expert outputs per (token, slot) over all global experts, summed shared expert first, then in top-k order num_tokens, num_topk = topk_idx.shape slot_y = torch.zeros((num_tokens, num_topk, xs[0].size(1)), dtype=dtype, device='cuda') @@ -143,7 +143,9 @@ def _moe_reference(expert, xs: Tuple, topk_idx: torch.Tensor, topk_weights: torc topk_weights[tokens, slots].to(dtype)).to(dtype) y = torch.zeros((num_tokens, xs[0].size(1)), dtype=dtype, device='cuda') if shared_weights is not None: - y += expert(*xs, *shared_weights, torch.ones(num_tokens, dtype=dtype, device='cuda')).to(dtype) + if shared_expert_weights is None: + shared_expert_weights = torch.ones(num_tokens, device='cuda') + y += expert(*xs, *shared_weights, shared_expert_weights.to(dtype)).to(dtype) for i in range(num_topk): y += slot_y[:, i] return y @@ -226,6 +228,12 @@ def _calc_token_diff(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: (7, 7168, 3072, 8, 8, 0, 1, 'hot', dict(activation='swiglustep')), (63, 4096, 256, 2, 1, 0, 0.5, 'random', dict(activation='swiglustep', **_EXACT)), (4096, 2048, 1536, 48, 6, 0, None, 'random', dict(activation='swiglustep')), + (0, 2048, 256, 8, 6, 1, 10, 'random', dict(shared_expert_weights=True)), + (1, 5120, 1536, 48, 8, 1, None, 'masked', dict(shared_expert_weights=True)), + (63, 4096, 256, 2, 1, 2, None, 'hot', dict(shared_expert_weights=True, **_EXACT)), + (129, 7168, 3072, 48, 8, 1, 10, 'empty', dict(shared_expert_weights=True)), + (1000, 2048, 2304, 8, 6, 2, 10, 'random', dict(shared_expert_weights=True)), + (8192, 7168, 3072, 48, 6, 1, 10, 'masked', dict(shared_expert_weights=True)), ) @@ -236,6 +244,8 @@ def _get_sweep_args(args: argparse.Namespace, num_ranks: int): activation_clamp, routing, extra in cases: if get_arch_major() != 9 and (extra.get('subnormal_probe') or (extra.get('exact_inputs') and not args.emu_reference)): continue + if get_arch_major() == 9 and extra.get('shared_expert_weights'): + continue yield argparse.Namespace(**{ **_CASE_DEFAULTS, **vars(args), 'num_tokens': num_tokens, 'num_max_tokens_per_rank': max(num_tokens, 1), @@ -243,7 +253,7 @@ def _get_sweep_args(args: argparse.Namespace, num_ranks: int): 'num_experts': num_experts_per_rank * num_ranks, 'num_topk': num_topk, 'num_shared_experts': num_shared_experts, 'activation_clamp': activation_clamp, 'routing': 'random' if routing == 'masked' else routing, 'masked_ratio': 0.3 if routing == 'masked' else 0, - **extra}) + 'shared_expert_weights': False, **extra}) def _copy_fp8_sf(dst: torch.Tensor, src: torch.Tensor, num_tokens: int) -> None: @@ -293,6 +303,7 @@ def test_case(rank_idx: int, num_ranks: int, group: dist.ProcessGroup, args: arg assert not use_emu_reference or (not is_bf16xbf16 and args.activation in ('swiglu', 'swiglustep')) assert not is_exact_inputs or (use_emu_reference and not args.fast_math) assert not args.subnormal_probe or arch_major == 9 + assert not args.shared_expert_weights or (num_shared_experts > 0 and arch_major == 10 and args.activation != 'situ') # Allocate symmetric memory buffer = deep_gemm.get_symm_buffer_for_mega_moe( @@ -367,6 +378,7 @@ def create_inputs(): global cumulative_local_expert_recv_stats_fused, cumulative_local_expert_recv_stats_baseline global initial_cumulative_local_expert_recv_stats_fused, initial_cumulative_local_expert_recv_stats_baseline global ref_x, ref_x_hi, all_l1_weights, all_l2_weights, ref_shared_weights, hi_shared_weights + global shared_expert_weights x = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device='cuda') if is_bf16xbf16: l1_weights = torch.randn( @@ -383,6 +395,11 @@ def create_inputs(): topk_weights = scores.gather(1, topk_idx) if is_exact_inputs: topk_weights = _exp2(torch.randint(126, 129, topk_weights.shape, device='cuda')) + # Per-token shared expert weights in (0, 1), like Qwen's `sigmoid(shared_expert_gate(x))` + shared_expert_weights = None + if args.shared_expert_weights: + shared_expert_weights = _exp2(torch.randint(125, 127, (num_tokens, ), device='cuda')) if is_exact_inputs else \ + torch.rand(num_tokens, dtype=torch.float, device='cuda') cumulative_local_expert_recv_stats_fused = torch.randint( 0, 100, (num_experts_per_rank, ), dtype=torch.int, device='cuda') cumulative_local_expert_recv_stats_baseline = cumulative_local_expert_recv_stats_fused.clone() @@ -525,6 +542,8 @@ def run_fused(): shared_l1_weights=transformed_shared_l1_weights, shared_l2_weights=transformed_shared_l2_weights ) + if shared_expert_weights is not None: + kernel_kwargs.update(shared_expert_weights=shared_expert_weights) (deep_gemm.bf16_mega_moe if is_bf16xbf16 else deep_gemm.fp8_fp4_mega_moe)(**kernel_kwargs) return y, cumulative_local_expert_recv_stats_fused @@ -537,7 +556,7 @@ def run_fused(): dist_print(f' > Tokens: {num_tokens}/{num_max_tokens_per_rank}', once_in_node=True) dist_print(f' > Hidden: {hidden}', once_in_node=True) dist_print(f' > Intermediate: {intermediate_hidden}', once_in_node=True) - dist_print(f' > Shared experts: {num_shared_experts}', once_in_node=True) + dist_print(f' > Shared experts: {num_shared_experts}{" (per-token weights)" if args.shared_expert_weights else ""}', once_in_node=True) dist_print(f' > Experts: {num_topk}/{num_experts}', once_in_node=True) dist_print(f' > Block M: {block_m}', once_in_node=True) dist_print(f' > Buffer: {buffer.buffer.nbytes / 2 ** 30:.3f} GiB', once_in_node=True) @@ -668,7 +687,7 @@ def get_baseline_shared_bias() -> Optional[torch.Tensor]: l1_out = torch.empty((num_tokens, shared_intermediate_hidden * 2), dtype=torch.bfloat16, device='cuda') deep_gemm.bf16_gemm_nt(x, shared_l1_weights, l1_out) l2_in = tilelang_ops.swiglu_apply_weight_to_fp8( - x=l1_out, topk_weights=None, + x=l1_out, topk_weights=shared_expert_weights, avail_tokens=None, num_per_channels=128, use_col_major_scales=True, clamp_value=args.activation_clamp, fast_math=bool(args.fast_math), @@ -680,7 +699,7 @@ def get_baseline_shared_bias() -> Optional[torch.Tensor]: l1_out = torch.empty((num_tokens, shared_intermediate_hidden * 2), dtype=torch.bfloat16, device='cuda') deep_gemm.fp8_gemm_nt(shared_x, shared_l1_weights, l1_out, recipe=(1, 1, 32), disable_ue8m0_cast=True) l2_in = tilelang_ops.swiglu_apply_weight_to_fp8( - x=l1_out, topk_weights=None, + x=l1_out, topk_weights=shared_expert_weights, avail_tokens=None, num_per_channels=32, use_col_major_scales=True, clamp_value=args.activation_clamp, fast_math=bool(args.fast_math), @@ -743,10 +762,10 @@ def get_num_recv_tokens_per_expert() -> torch.Tensor: assert torch.equal(fused_stats, initial_cumulative_local_expert_recv_stats_fused + get_num_recv_tokens_per_expert()) ref_y = _moe_reference( functools.partial(_emu_expert, gran_k=gran_k, args=args), ref_x, topk_idx, topk_weights, - all_l1_weights, all_l2_weights, ref_shared_weights, torch.float).bfloat16() + all_l1_weights, all_l2_weights, ref_shared_weights, shared_expert_weights, torch.float).bfloat16() hi_y = _moe_reference( functools.partial(_hi_expert, args=args), (ref_x_hi, ), topk_idx, topk_weights, - all_l1_weights, all_l2_weights, hi_shared_weights, torch.double) + all_l1_weights, all_l2_weights, hi_shared_weights, shared_expert_weights, torch.double) diff, hi_diff = calc_diff(fused_y, ref_y), calc_diff(fused_y, hi_y) max_token_diff = max(_calc_token_diff(fused_y, ref_y).tolist(), default=0) dist_print(f' > Accuracy (rank {rank_idx}): calc_diff={diff:.3e}, max_token_diff={max_token_diff:.3e}, hi_diff={hi_diff:.3e}') @@ -760,8 +779,10 @@ def get_num_recv_tokens_per_expert() -> torch.Tensor: dist_print(f' > Correctness test #{i + 1}/{num_correctness_tests} passed', once_in_node=True) # Replays of one CUDA graph alternate two input sets, and must match the eager launches bit for bit - graph_inputs = [*x, topk_weights] - input_sets = [[t.clone() for t in graph_inputs], [t.view(torch.uint8).roll(1, 0).view(t.dtype) for t in x] + [-topk_weights]] + weight_inputs = [topk_weights] + ([shared_expert_weights] if shared_expert_weights is not None else []) + graph_inputs = [*x, *weight_inputs] + input_sets = [[t.clone() for t in graph_inputs], + [t.view(torch.uint8).roll(1, 0).view(t.dtype) for t in x] + [-t for t in weight_inputs]] eager_ys = [] for input_set in input_sets: for t, s in zip(graph_inputs, input_set): @@ -894,6 +915,8 @@ def test(local_rank: int, num_local_ranks: int, args: argparse.Namespace): parser.add_argument('--hidden', type=int, default=7168, help='Hidden size') parser.add_argument('--intermediate-hidden', type=int, default=3072, help='Intermediate hidden size') parser.add_argument('--num-shared-experts', type=int, default=1, help='Number of shared experts (use 0 to disable)') + parser.add_argument('--shared-expert-weights', action='store_true', + help='Scale the shared experts per token, like Qwen\'s `sigmoid(shared_expert_gate(x))` (SM100 only)') parser.add_argument('--activation', type=str, default='swiglu', choices=('swiglu', 'swiglustep', 'situ')) parser.add_argument('--activation-clamp', type=float, default=10, help='SwiGLU clamp value') parser.add_argument('--activation-alpha', type=float, default=1.0,