Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 45 additions & 11 deletions csrc/apis/mega_moe.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,8 @@ static void fp8_fp4_mega_moe(
const bool& fast_math,
const float& activation_alpha,
const float& activation_beta,
const std::optional<int64_t>& num_tokens_for_block_m
const std::optional<int64_t>& num_tokens_for_block_m,
const std::optional<torch::stable::Tensor>& shared_expert_weights_opt
) {
const auto [l1_weights, l1_weights_sf] = l1_weights_tuple;
const auto [l2_weights, l2_weights_sf] = l2_weights_tuple;
Expand Down Expand Up @@ -398,6 +399,20 @@ static void fp8_fp4_mega_moe(
DG_HOST_ASSERT(cumulative_local_expert_recv_stats->is_contiguous());
}

// Check shared expert weights, e.g. Qwen's `sigmoid(shared_expert_gate(x))`, applied where routed tokens
// apply their top-k weight
const float* shared_expert_weights = nullptr;
if (shared_expert_weights_opt.has_value()) {
const auto& weights = shared_expert_weights_opt.value();
DG_HOST_ASSERT(weights.scalar_type() == torch::headeronly::ScalarType::Float and weights.is_contiguous());
DG_HOST_ASSERT(weights.dim() == 1 and weights.size(0) == num_tokens);
DG_HOST_ASSERT(weights.is_cuda() and weights.get_device_index() == y.get_device_index());
DG_HOST_ASSERT(num_shared_experts > 0);
// NOTES: only the SM100 SwiGLU kernel implements shared expert weights
DG_HOST_ASSERT(arch_major == 10 and activation != "situ");
shared_expert_weights = weights.const_data_ptr<float>();
}

// Check buffer bytes
const auto num_ranks = static_cast<int>(sym_buffer_ptrs.size());
const auto num_experts_ = num_experts_per_rank * num_ranks;
Expand Down Expand Up @@ -445,7 +460,8 @@ static void fp8_fp4_mega_moe(
activation_alpha, activation_beta, fast_math);
} else {
launch(sm100_fp8_fp4_mega_moe,
activation_clamp, activation == "swiglustep", activation_alpha, activation_beta, fast_math);
activation_clamp, activation == "swiglustep", activation_alpha, activation_beta, fast_math,
shared_expert_weights);
}
} else if (arch_major == 9) {
DG_HOST_ASSERT(activation != "situ");
Expand Down Expand Up @@ -694,7 +710,8 @@ static void bf16_mega_moe(
const float& activation_alpha,
const float& activation_beta,
const std::optional<torch::stable::Tensor>& post_expert_norm_gamma_opt,
const float& post_expert_norm_eps
const float& post_expert_norm_eps,
const std::optional<torch::stable::Tensor>& shared_expert_weights_opt
) {
// Config checks
const auto num_tokens = static_cast<int>(y.size(0));
Expand Down Expand Up @@ -780,6 +797,18 @@ static void bf16_mega_moe(
DG_HOST_ASSERT(torch_compat::nbytes(sym_buffer) >= static_cast<size_t>(layout_info.num_bytes));
DG_HOST_ASSERT(num_experts == num_experts_);

// Check shared expert weights, e.g. Qwen's `sigmoid(shared_expert_gate(x))`, applied where routed tokens
// apply their top-k weight
const float* shared_expert_weights = nullptr;
if (shared_expert_weights_opt.has_value()) {
const auto& weights = shared_expert_weights_opt.value();
DG_HOST_ASSERT(weights.scalar_type() == torch::headeronly::ScalarType::Float and weights.is_contiguous());
DG_HOST_ASSERT(weights.dim() == 1 and weights.size(0) == num_tokens);
DG_HOST_ASSERT(weights.is_cuda() and weights.get_device_index() == y.get_device_index());
DG_HOST_ASSERT(num_shared_experts > 0);
shared_expert_weights = weights.const_data_ptr<float>();
}

const void* post_expert_norm_gamma_ptr = nullptr;
if (post_expert_norm_gamma_opt.has_value()) {
const auto& gamma = post_expert_norm_gamma_opt.value();
Expand Down Expand Up @@ -814,7 +843,8 @@ static void bf16_mega_moe(
hidden, intermediate_hidden,
activation_clamp, activation == "swiglustep", activation_alpha, activation_beta,
fast_math,
post_expert_norm_gamma_ptr, post_expert_norm_eps);
post_expert_norm_gamma_ptr, post_expert_norm_eps,
shared_expert_weights);
} else {
DG_HOST_UNREACHABLE("Unsupported architecture");
}
Expand Down Expand Up @@ -884,7 +914,8 @@ static void fp8_fp4_mega_moe(
const bool& fast_math,
const double& activation_alpha,
const double& activation_beta,
const std::optional<int64_t>& num_tokens_for_block_m) {
const std::optional<int64_t>& num_tokens_for_block_m,
const std::optional<torch::stable::Tensor>& shared_expert_weights_opt) {
mega::fp8_fp4_mega_moe(
y, {l1_weights_tuple, l1_weights_tuple_sf}, {l2_weights_tuple, l2_weights_tuple_sf},
make_optional_tensor_pair(shared_l1_weights_tuple_opt, shared_l1_weights_tuple_opt_sf),
Expand All @@ -893,7 +924,8 @@ static void fp8_fp4_mega_moe(
mega::checked_int(rank_idx), mega::checked_int(num_max_tokens_per_rank),
mega::checked_int(num_experts), mega::checked_int(num_topk), checked_recipe(recipe),
activation, activation_clamp_opt, fast_math,
static_cast<float>(activation_alpha), static_cast<float>(activation_beta), num_tokens_for_block_m);
static_cast<float>(activation_alpha), static_cast<float>(activation_beta), num_tokens_for_block_m,
shared_expert_weights_opt);
}

template <bool use_trtllm_weights>
Expand All @@ -916,15 +948,17 @@ static void bf16_mega_moe(
const double& activation_alpha,
const double& activation_beta,
const std::optional<torch::stable::Tensor>& post_expert_norm_gamma_opt,
const double& post_expert_norm_eps) {
const double& post_expert_norm_eps,
const std::optional<torch::stable::Tensor>& shared_expert_weights_opt) {
mega::bf16_mega_moe<use_trtllm_weights>(
y, l1_weights, l2_weights, shared_l1_weights_opt, shared_l2_weights_opt,
cumulative_local_expert_recv_stats, sym_buffer, sym_buffer_ptrs,
mega::checked_int(rank_idx), mega::checked_int(num_max_tokens_per_rank),
mega::checked_int(num_experts), mega::checked_int(num_topk), activation,
activation_clamp_opt, fast_math,
static_cast<float>(activation_alpha), static_cast<float>(activation_beta),
post_expert_norm_gamma_opt, static_cast<float>(post_expert_norm_eps));
post_expert_norm_gamma_opt, static_cast<float>(post_expert_norm_eps),
shared_expert_weights_opt);
}

template <bool use_trtllm_weights>
Expand Down Expand Up @@ -989,9 +1023,9 @@ STABLE_TORCH_LIBRARY_FRAGMENT(deep_gemm, m) {
m.def("get_symm_buffer_size_for_mega_moe(int num_ranks, int num_experts, int num_max_tokens_per_rank, int num_topk, int hidden, int intermediate_hidden, str mma_type, str activation, int num_shared_experts) -> (int, int[])");
m.def("_slice_symm_buffer_for_mega_moe(Tensor(a) buffer, int[] layout_info) -> (Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a), Tensor(a))");

m.def("fp8_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_tuple_opt, Tensor? shared_l1_weights_tuple_opt_sf, Tensor? shared_l2_weights_tuple_opt, Tensor? shared_l2_weights_tuple_opt_sf, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, int? num_tokens_for_block_m) -> ()");
m.def("bf16_mega_moe(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()");
m.def("bf16_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()");
m.def("fp8_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_tuple_opt, Tensor? shared_l1_weights_tuple_opt_sf, Tensor? shared_l2_weights_tuple_opt, Tensor? shared_l2_weights_tuple_opt_sf, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, int? num_tokens_for_block_m, Tensor? shared_expert_weights_opt=None) -> ()");
m.def("bf16_mega_moe(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps, Tensor? shared_expert_weights_opt=None) -> ()");
m.def("bf16_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights, Tensor l2_weights, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, str activation, float? activation_clamp_opt, bool fast_math, float activation_alpha, float activation_beta, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps, Tensor? shared_expert_weights_opt=None) -> ()");
m.def("fp4_fp4_mega_moe(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor? x_bf16_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, Tensor? l1_alphas_opt, Tensor? l2_alphas_opt, Tensor? a2_scales_opt, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()");
m.def("fp4_fp4_mega_moe_trtllm(Tensor(a!) y, Tensor l1_weights_tuple, Tensor l1_weights_tuple_sf, Tensor l2_weights_tuple, Tensor l2_weights_tuple_sf, Tensor? shared_l1_weights_opt, Tensor? shared_l2_weights_opt, Tensor? x_bf16_opt, Tensor(b!)? cumulative_local_expert_recv_stats, Tensor(c!) sym_buffer, int[] sym_buffer_ptrs, int rank_idx, int num_max_tokens_per_rank, int num_experts, int num_topk, int[3] recipe, str activation, float? activation_clamp_opt, bool fast_math, Tensor? l1_alphas_opt, Tensor? l2_alphas_opt, Tensor? a2_scales_opt, Tensor? post_expert_norm_gamma_opt, float post_expert_norm_eps) -> ()");
}
Expand Down
6 changes: 5 additions & 1 deletion csrc/jit_kernels/impls/sm100_bf16_mega_moe.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ static void sm100_bf16_mega_moe(
const float& activation_alpha,
const float& activation_beta,
const bool& fast_math,
const void* post_expert_norm_gamma, const float& post_expert_norm_eps
const void* post_expert_norm_gamma, const float& post_expert_norm_eps,
const float* shared_expert_weights
) {
const auto num_ranks = static_cast<int>(sym_buffer_ptrs.size());
const auto num_experts = num_experts_per_rank * num_ranks;
Expand Down Expand Up @@ -166,6 +167,7 @@ static void __instantiate_kernel() {{
{},
{},
{},
{},
{}
>);
}};
Expand All @@ -186,6 +188,7 @@ static void __instantiate_kernel() {{
to_string(activation_beta),
fast_math ? "true" : "false",
post_expert_norm_gamma != nullptr ? "true" : "false",
shared_expert_weights != nullptr ? "true" : "false",
use_trtllm_weights ? "true" : "false"));

// Launch
Expand All @@ -199,6 +202,7 @@ static void __instantiate_kernel() {{
y.mutable_data_ptr(),
cumulative_local_expert_recv_stats_ptr,
num_tokens,
shared_expert_weights,
post_expert_norm_gamma, post_expert_norm_eps,
layout::SymBuffer<>(sym_buffer_ptrs, rank_idx),
tensor_map_l1_acts,
Expand Down
6 changes: 5 additions & 1 deletion csrc/jit_kernels/impls/sm100_fp8_fp4_mega_moe.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ static void sm100_fp8_fp4_mega_moe(
const bool& clamp_after_silu,
const float& activation_alpha,
const float& activation_beta,
const bool& fast_math
const bool& fast_math,
const float* shared_expert_weights
) {
const auto num_ranks = static_cast<int>(sym_buffer_ptrs.size());
const auto num_experts = num_experts_per_rank * num_ranks;
Expand Down Expand Up @@ -175,6 +176,7 @@ static void __instantiate_kernel() {{
{},
{},
{},
{},
{}
>);
}};
Expand All @@ -196,6 +198,7 @@ static void __instantiate_kernel() {{
to_string(activation_alpha),
to_string(activation_beta),
fast_math ? "true" : "false",
shared_expert_weights != nullptr ? "true" : "false",
to_string(l1_weights.scalar_type())));

// Launch
Expand All @@ -209,6 +212,7 @@ static void __instantiate_kernel() {{
y.mutable_data_ptr(),
cumulative_local_expert_recv_stats_ptr,
num_tokens,
shared_expert_weights,
layout::SymBuffer<>(sym_buffer_ptrs, rank_idx),
tensor_map_l1_acts,
tensor_map_l1_acts_sf,
Expand Down
6 changes: 6 additions & 0 deletions deep_gemm/_C.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,7 @@ def fp8_fp4_mega_moe(
activation_alpha,
activation_beta,
num_tokens_for_block_m=None,
shared_expert_weights_opt=None,
):
return _torch_ops.fp8_fp4_mega_moe(
y,
Expand All @@ -539,6 +540,7 @@ def fp8_fp4_mega_moe(
activation_alpha,
activation_beta,
num_tokens_for_block_m,
shared_expert_weights_opt,
)


Expand All @@ -562,6 +564,7 @@ def bf16_mega_moe(
activation_beta,
post_expert_norm_gamma_opt=None,
post_expert_norm_eps=0.0,
shared_expert_weights_opt=None,
):
return _torch_ops.bf16_mega_moe(
y,
Expand All @@ -583,6 +586,7 @@ def bf16_mega_moe(
activation_beta,
post_expert_norm_gamma_opt,
post_expert_norm_eps,
shared_expert_weights_opt,
)


Expand All @@ -606,6 +610,7 @@ def bf16_mega_moe_trtllm(
activation_beta,
post_expert_norm_gamma_opt=None,
post_expert_norm_eps=0.0,
shared_expert_weights_opt=None,
):
return _torch_ops.bf16_mega_moe_trtllm(
y,
Expand All @@ -627,6 +632,7 @@ def bf16_mega_moe_trtllm(
activation_beta,
post_expert_norm_gamma_opt,
post_expert_norm_eps,
shared_expert_weights_opt,
)


Expand Down
12 changes: 12 additions & 0 deletions deep_gemm/include/deep_gemm/impls/sm100_bf16_mega_moe.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ template <
float kActivationBeta,
bool kFastMath,
bool kWithPostExpertNorm,
bool kWithSharedExpertWeights,
bool kUseTrtllmWeights = false,
bool kHasShared = (kNumSharedExperts > 0),
uint32_t L1_SHAPE_N = kIntermediateHidden * 2,
Expand All @@ -58,6 +59,7 @@ CUTLASS_GLOBAL __launch_bounds__(kNumThreads, 1) void
sm100_bf16_mega_moe_impl(void* y,
int* cumulative_local_expert_recv_stats,
const uint32_t num_tokens,
const float* shared_expert_weights,
const float* post_expert_norm_gamma,
const float post_expert_norm_eps,
const __grid_constant__ layout::SymBuffer<kNumRanks> sym_buffer,
Expand Down Expand Up @@ -141,6 +143,7 @@ sm100_bf16_mega_moe_impl(void* y,
DG_STATIC_ASSERT(not kUseTrtllmWeights or (not kHasShared and BLOCK_K == 64 and LOAD_BLOCK_N % 32 == 0),
"TRT-LLM BF16 weights require no shared experts, 64-element K blocks and 32-row aligned tiles");
DG_STATIC_ASSERT(BLOCK_M % 16 == 0, "Invalid block M");
DG_STATIC_ASSERT(not kWithSharedExpertWeights or kHasShared, "Shared expert weights require fused shared experts");
DG_STATIC_ASSERT(BLOCK_N == LAYOUT_AD_M, "Invalid block N");

// Swizzle configs
Expand Down Expand Up @@ -943,6 +946,15 @@ sm100_bf16_mega_moe_impl(void* y,
.get_data_buffer(ring_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx)
.template get_base_ptr<float>();
}
// Optional per-token shared-expert weight (e.g. Qwen's `sigmoid(shared_expert_gate(x))`),
// applied where routed tokens apply their top-k weight. Shared rows are local token indices.
if constexpr (kWithSharedExpertWeights) {
if (task_info.is_shared() and (j * ATOM_M) % 32 == 0 and
(WG_BLOCK_M % 32 == 0 or j * ATOM_M + lane_idx < WG_BLOCK_M)) {
const uint32_t token_idx = pool_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx;
stored_cached_weight = token_idx < num_tokens ? __ldg(shared_expert_weights + token_idx) : 0.0f;
}
}

// Load weights from register cache
const float2 weights = {
Expand Down
12 changes: 12 additions & 0 deletions deep_gemm/include/deep_gemm/impls/sm100_fp8_fp4_mega_moe.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ template <
float kActivationAlpha,
float kActivationBeta,
bool kFastMath,
bool kWithSharedExpertWeights,
typename weight_dtype_t,
bool kHasShared = (kNumSharedExperts > 0),
uint32_t L1_SHAPE_N = kIntermediateHidden * 2,
Expand All @@ -60,6 +61,7 @@ CUTLASS_GLOBAL __launch_bounds__(kNumThreads, 1) void
sm100_fp8_fp4_mega_moe_impl(void* y,
int* cumulative_local_expert_recv_stats,
const uint32_t num_tokens,
const float* shared_expert_weights,
const __grid_constant__ layout::SymBuffer<kNumRanks> sym_buffer,
const __grid_constant__ cute::TmaDescriptor tensor_map_l1_acts,
const __grid_constant__ cute::TmaDescriptor tensor_map_l1_acts_sf,
Expand Down Expand Up @@ -94,6 +96,7 @@ sm100_fp8_fp4_mega_moe_impl(void* y,
DG_STATIC_ASSERT(kNumNonEpilogueThreads == 128, "Invalid number of MMA non-epilogue threads");
DG_STATIC_ASSERT(kNumEpilogueThreads % 128 == 0, "Invalid number of MMA epilogue and combine threads");
DG_STATIC_ASSERT(kNumExperts % kNumRanks == 0, "Invalid number of experts or ranks");
DG_STATIC_ASSERT(not kWithSharedExpertWeights or kHasShared, "Shared expert weights require fused shared experts");

// Thread indices
const bool is_leader_cta = cute::block_rank_in_cluster() == 0;
Expand Down Expand Up @@ -1101,6 +1104,15 @@ sm100_fp8_fp4_mega_moe_impl(void* y,
.get_data_buffer(ring_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx)
.template get_base_ptr<float>();
}
// Optional per-token shared-expert weight (e.g. Qwen's `sigmoid(shared_expert_gate(x))`),
// applied where routed tokens apply their top-k weight. Shared rows are local token indices.
if constexpr (kWithSharedExpertWeights) {
if (task_info.is_shared() and (j * ATOM_M) % 32 == 0 and
(WG_BLOCK_M % 32 == 0 or j * ATOM_M + lane_idx < WG_BLOCK_M)) {
const uint32_t token_idx = pool_m_idx + epilogue_wg_idx * WG_BLOCK_M + j * ATOM_M + lane_idx;
stored_cached_weight = token_idx < num_tokens ? __ldg(shared_expert_weights + token_idx) : 0.0f;
}
}

// Load weights from register cache
const float2 weights = {
Expand Down
Loading