From eb16d607fe961307b8820892a5d907108d7d02e2 Mon Sep 17 00:00:00 2001 From: xiangch Date: Tue, 3 Nov 2026 10:23:05 +0000 Subject: [PATCH] perf(ep/v2): AsyncLL slot assignment by block, receive copy over the whole grid Two dispatch passes of the AsyncLL family were well short of what their work needs. Measured at sglang's DeepSeek-V4 shape (EP8, hidden 7168, top-6 of 384 experts, 4096 tokens/rank, fp8 dispatch with 224 B of e8m0 scales, sdma, early notification), mean of 8 ranks, 3 interleaved runs: before after dispatch slotassign 58.6 us 6.6 us dispatch recvcopy 65.4 us 53.9 us SlotAssign claimed each entry's slot with its own atomicAdd on one of worldSize counters: ~18k atomics on 8 addresses, for a pass that moves a few hundred KB of routing. Warps now count their claims per destination PE with a ballot and take a run inside the block through LDS; one thread per PE then claims the block's run from the global counter, ~200 global atomics in all. Which slot an entry gets was already atomic order, and RecvCopy records each token's origin, so nothing downstream depends on it. RecvCopy split its grid evenly across source PEs, so the PE that sent the most set the kernel's length while the other groups idled. The whole grid now walks the concatenated output and finds each row's source PE from the prefix sum with one ballot. The payload copy keeps four 16 B loads in flight per lane (the source is what SDMA just wrote, so it comes from HBM), and the short metadata fields go a 4 B word per lane, with a byte fallback for unaligned fields, instead of through WarpCopy's byte tail. The other copy passes already run at 4.5-5 TB/s and are unchanged. Two SendCopy variants were tried and dropped: reading each token once for all its destinations (+19%) and a 4-deep copy (+25%). test_asyncll passes on 8x MI355X for sdma (legacy, early, early + fused receive) and lsa. The fp8 benchmark above also checks every received row, index, weight and scale byte for byte against its source, and the combined output against the expected sum. Co-Authored-By: Claude --- .../dispatch_combine_v2/ep_asyncll_spec.hpp | 8 +- .../dispatch_combine_v2/ep_asyncll_kernel.hpp | 180 ++++++++++++------ .../dispatch_combine_v2/ep_asyncll_spec.cpp | 7 +- 3 files changed, 134 insertions(+), 61 deletions(-) diff --git a/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp b/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp index 0cd0882bc2..e580b2dd83 100644 --- a/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp +++ b/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp @@ -84,10 +84,10 @@ std::string EpAsyncLLRenderSource(const EpInterNodeCfg& cfg, EpAsyncLLKernel kin mori::jit::v2::LaunchGeometry EpAsyncLLGeometry(const EpInterNodeCfg& cfg, EpAsyncLLKernel kind); // Grids that divide the work per destination rank must be an exact multiple of -// worldSize: DispatchRecvCopy derives `blocksPerPe = gridDim.x / worldSize` and -// then `destPe = blockIdx.x / blocksPerPe`, so a remainder leaves the tail blocks -// decoding to a PE that does not exist. v1 aligns the same way at its launch -// site (`mp - mp % worldSize`). +// worldSize: the LSA transfer passes derive `destPe = globalWarpId / +// (globalWarpNum / worldSize)`, so a remainder leaves the tail warps decoding to +// a PE that does not exist. v1 aligns the same way at its launch site +// (`mp - mp % worldSize`). constexpr int EpAsyncLLAlignGrid(int grid, int worldSize) { const int aligned = grid - (grid % worldSize); return aligned > 0 ? aligned : worldSize; diff --git a/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp b/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp index aa6f795b5e..5c0a20c31e 100644 --- a/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp +++ b/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp @@ -180,6 +180,21 @@ __device__ __forceinline__ void PostPayloadAndNotify( } } +// Copy a short field one 4 B word per lane, falling back to one byte per lane +// unless the length and both ends are 4 B aligned. At the shapes in use they are +// (hidden and scale rows are multiples of 4), but the record stride is the sum +// of every field, so an odd scale row would misalign every record after it. +__device__ __forceinline__ void LaneCopyWords(uint8_t* dst, const uint8_t* src, size_t bytes) { + const int lane = threadIdx.x & (warpSize - 1); + if (((bytes | reinterpret_cast(dst) | reinterpret_cast(src)) & 3) == 0) { + for (size_t b = static_cast(lane) * 4; b < bytes; + b += static_cast(warpSize) * 4) + *reinterpret_cast(dst + b) = *reinterpret_cast(src + b); + } else { + for (size_t b = lane; b < bytes; b += warpSize) dst[b] = src[b]; + } +} + } // namespace asyncll /* ---------------------------------------------------------------------------------------------- */ @@ -188,52 +203,99 @@ __device__ __forceinline__ void PostPayloadAndNotify( // Routing only: no payload moves here. Groups of numExpertPerToken lanes inside a // warp cooperate on one token, each lane holding its own expert's destination, so // the deduplication is a register shuffle rather than a reload. +// +// Slots are claimed per BLOCK, not per entry. Every surviving entry needs the +// next slot of its destination PE, and there are only worldSize counters: one +// atomicAdd per entry put ~18k atomics on 8 addresses at EP8, top-6, 4096 tokens +// and the pass took ~58 us for a few hundred KB of routing. Here each warp claims +// its run inside the block (one LDS atomic per destination PE it touches), then +// one thread per PE claims the block's whole run from the global counter: ~6.5 us +// at that shape. The block walks the tokens in lockstep so every __syncthreads +// is reached by all of its threads. +// +// Which slot an entry gets was never deterministic (it was atomic order before), +// and nothing downstream depends on it: RecvCopy compacts each PE's slots in slot +// order and records where each token came from. template __device__ void EpDispatchAsyncLLSlotAssign_body(EpDispatchCombineArgs args) { DEF_COMMON_VARS; + static_assert(kConfig.worldSize <= 64, + "slot claims are tracked in one 64-entry LDS array per destination PE"); constexpr int numEpt = kConfig.numExpertPerToken; const int tokensPerWarp = warpSize / numEpt; + const int tokensPerBlock = warpNum * tokensPerWarp; const int expertIdx = laneId % numEpt; const int inWarpTokIdx = laneId / numEpt; const int baseLane = inWarpTokIdx * numEpt; + const unsigned long long lanesBelow = (1ull << laneId) - 1ull; + + __shared__ index_t blockClaims[64]; + __shared__ index_t blockBase[64]; + + for (int blockTokBase = blockId * tokensPerBlock; blockTokBase < args.curRankNumToken; + blockTokBase += blockNum * tokensPerBlock) { + if (threadIdx.x < kConfig.worldSize) blockClaims[threadIdx.x] = 0; + __syncthreads(); - for (int warpTokBase = globalWarpId * tokensPerWarp; warpTokBase < args.curRankNumToken; - warpTokBase += globalWarpNum * tokensPerWarp) { - const int tokenId = warpTokBase + inWarpTokIdx; + const int tokenId = blockTokBase + warpId * tokensPerWarp + inWarpTokIdx; // warpSize need not be a multiple of numEpt (64 % 6 == 4 at top-k 6): the // trailing lanes alias the next warp's first token, so one entry would get two // slots and the orphan -- never written by SendCopy -- is read as a token by // the receiver. The test at top-k 6/9/10/12 exists for this. // - // Dropping a whole group keeps the shuffle below warp-coherent: inWarpTokIdx - // is group-uniform, so a group is either entirely in or entirely out. - if ((inWarpTokIdx >= tokensPerWarp) || (tokenId >= args.curRankNumToken)) continue; - + // Those lanes and the ones past the last token stay in the loop as inactive + // rather than continuing: the ballots and shuffles below, and the block + // barriers, need every lane. + const bool active = (inWarpTokIdx < tokensPerWarp) && (tokenId < args.curRankNumToken); const int i = tokenId * numEpt + expertIdx; - const index_t destExpert = args.tokenIndices[i]; // A negative expert id is the caller's "drop this slot" sentinel. + const index_t destExpert = active ? args.tokenIndices[i] : static_cast(-1); const bool dropped = (destExpert < 0); const index_t destPe = dropped ? 0 : (destExpert / config.numExpertPerRank); // Deduplicate by shuffling the earlier lanes' destinations out of their - // registers. Every lane runs the same trip count so __shfl stays uniform -- - // an early exit here is what makes it read an inactive lane. + // registers. inWarpTokIdx is group-uniform, so a group is entirely active or + // entirely not, and an inactive group's result is discarded. The mask keeps + // the trailing lanes' source lane inside the wave. bool isDuplicate = false; for (int j = 0; j < numEpt - 1; ++j) { - const index_t prevPe = __shfl(destPe, baseLane + j); - const bool prevDropped = __shfl(static_cast(dropped), baseLane + j) != 0; + const int src = (baseLane + j) & (warpSize - 1); + const index_t prevPe = __shfl(destPe, src); + const bool prevDropped = __shfl(static_cast(dropped), src) != 0; if ((j < expertIdx) && !prevDropped) isDuplicate = isDuplicate || (prevPe == destPe); } // An out-of-range expert id (an EPLB physical id past worldSize*numExpertPerRank) // would index destPeTokenCounter and SendBufSlotOffset out of bounds. Folded - // into the same null branch as the duplicate rather than skipped early, again - // to keep the shuffle above coherent. + // into the same null result as the duplicate rather than skipped early, again + // to keep the shuffles coherent. const bool peOutOfRange = (destPe < 0) || (destPe >= config.worldSize); - if (dropped || isDuplicate || peOutOfRange) { - args.dispDestTokIdMap[i] = NullSendBufSlotOffset(config); - } else { - const index_t destTokId = atomicAdd(args.destPeTokenCounter + destPe, 1); + const bool claims = active && !dropped && !isDuplicate && !peOutOfRange; + + // This lane's offset within the block's run for its PE. + index_t inBlockTokId = 0; +#pragma unroll + for (int pe = 0; pe < kConfig.worldSize; ++pe) { + const unsigned long long lanes = __ballot(claims && (destPe == pe)); + if (lanes) { + const int leader = __ffsll(static_cast(lanes)) - 1; + index_t warpBase = 0; + if (laneId == leader) + warpBase = atomicAdd(&blockClaims[pe], static_cast(__popcll(lanes))); + warpBase = __shfl(warpBase, leader); + if (claims && (destPe == pe)) + inBlockTokId = warpBase + static_cast(__popcll(lanes & lanesBelow)); + } + } + __syncthreads(); + if (threadIdx.x < kConfig.worldSize) { + const index_t n = blockClaims[threadIdx.x]; + blockBase[threadIdx.x] = n ? atomicAdd(args.destPeTokenCounter + threadIdx.x, n) : 0; + } + __syncthreads(); + + if (active) { + const index_t destTokId = claims ? blockBase[destPe] + inBlockTokId : 0; // The counter is cleared by RecvCopy, one per round, so a SECOND // dispatch_send with no recv between it and the first keeps climbing from // where the first left off. Unbounded, SendBufSlotOffset then encodes a @@ -246,10 +308,12 @@ __device__ void EpDispatchAsyncLLSlotAssign_body(EpDispatchCombineArgs args) { // right -- it is the host's call-order check that stops it being reachable // (dispatch_combine_op.py) -- but wrong-and-in-bounds beats corrupting a // peer's buffer, which is the one thing this kernel can still choose. - args.dispDestTokIdMap[i] = (destTokId < config.MaxNumTokensToSendPerRank()) + args.dispDestTokIdMap[i] = (claims && destTokId < config.MaxNumTokensToSendPerRank()) ? SendBufSlotOffset(config, destPe, destTokId) : NullSendBufSlotOffset(config); } + // blockClaims is reset at the top of the next round. + __syncthreads(); } } @@ -610,6 +674,12 @@ __device__ void EpDispatchAsyncLLRecvTransfer_body(EpDispatchCombineArgs args, // Unpack what arrived into the dispatch output, and build the two maps combine // needs. Per-PE output offsets come from a warp prefix sum rather than an atomic, // so every block derives the same layout independently. +// +// The whole grid walks the concatenated output rather than giving each source PE +// a fixed 1/worldSize of the blocks: with a static split the PE that sent the +// most sets the kernel's length while the other groups sit idle, and the PEs a +// rank receives from are rarely even. The owning PE of output row r is the last +// one whose exclusive offset is <= r, read off the prefix sum with one ballot. template __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { DEF_COMMON_VARS; @@ -621,11 +691,6 @@ __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { static_assert(kConfig.worldSize <= 64, "the per-PE offset prefix sum is one wavefront wide; worldSize must fit it"); - const int blocksPerPe = blockNum / npes; - if (blocksPerPe <= 0) return; - const int destPe = blockId / blocksPerPe; - if (destPe >= npes) return; - const uint8_t* recvCounts = args.reg(args.offRecvTokenNum)->template GetAs(); uint64_t myPeTokens = 0; @@ -640,56 +705,63 @@ __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { } uint64_t peExclusive = __shfl_up(inclusive, 1); if (laneId == 0) peExclusive = 0; - - const uint64_t peOffset = __shfl(peExclusive, destPe); - const uint64_t recvTokenNum = __shfl(myPeTokens, destPe); const uint64_t totalTokens = __shfl(inclusive, npes - 1); - const uint8_t* stagingPtr = (destPe != myPe) - ? args.reg(args.offDispatchInp)->template GetAs() - : args.reg(args.offStaging)->template GetAs(); - stagingPtr += static_cast(SendBufSlotOffset(config, destPe, 0)) * xferBytes; + const uint8_t* peerBase = args.reg(args.offDispatchInp)->template GetAs(); + // This rank's own tokens were never transferred; they are still in staging. + const uint8_t* selfBase = args.reg(args.offStaging)->template GetAs(); + uint8_t* outTokens = args.reg(args.offDispatchOut)->template GetAs(); + uint8_t* outIndices = args.reg(args.offOutIndices)->template GetAs(); + uint8_t* outWeights = args.reg(args.offDispatchOutWeights)->template GetAs(); + uint8_t* outScales = args.reg(args.offOutScales)->template GetAs(); - const int peWarps = blocksPerPe * warpNum; - const int localWarpId = (blockId % blocksPerPe) * warpNum + warpId; const index_t warpsPerToken = - (recvTokenNum > 0) ? static_cast((peWarps + recvTokenNum - 1) / recvTokenNum) : 1; + (totalTokens > 0) ? static_cast((globalWarpNum + totalTokens - 1) / totalTokens) : 1; const size_t hiddenBytesPerWarp = ((hiddenBytes + warpsPerToken - 1) / warpsPerToken + 15) & ~static_cast(15); - for (int i = localWarpId; i < static_cast(recvTokenNum) * warpsPerToken; i += peWarps) { - const index_t tokenId = i / warpsPerToken; + for (int i = globalWarpId; i < static_cast(totalTokens) * warpsPerToken; + i += globalWarpNum) { + const index_t destTokId = i / warpsPerToken; const index_t inTokenPartId = i % warpsPerToken; - const index_t destTokId = static_cast(peOffset) + tokenId; - + // Lane 0 always qualifies (its exclusive offset is 0), so the mask is never + // empty; a PE that sent nothing shares its offset with the next and is + // skipped by taking the highest qualifying lane. + const unsigned long long owners = + __ballot((laneId < npes) && (peExclusive <= static_cast(destTokId))); + const int srcPe = 63 - __clzll(static_cast(owners)); + const index_t tokenId = destTokId - static_cast(__shfl(peExclusive, srcPe)); + const uint8_t* record = + ((srcPe != myPe) ? peerBase : selfBase) + + (static_cast(SendBufSlotOffset(config, srcPe, 0)) + tokenId) * xferBytes; + + // Four 16 B loads in flight per lane: the source is what SDMA just wrote, so + // it comes from HBM, not cache, and one load per lane left it latency-bound. const size_t hiddenOffset = inTokenPartId * hiddenBytesPerWarp; if (hiddenOffset < hiddenBytes) { const size_t len = min(hiddenBytesPerWarp, hiddenBytes - hiddenOffset); - core::WarpCopy(args.reg(args.offDispatchOut)->template GetAs() + - destTokId * hiddenBytes + hiddenOffset, - stagingPtr + tokenId * xferBytes + hiddenOffset, len); + core::WarpCopy(outTokens + destTokId * hiddenBytes + hiddenOffset, + record + hiddenOffset, len); } if (inTokenPartId == 0) { - core::WarpCopy( - args.reg(args.offOutIndices)->template GetAs() + destTokId * indexBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes, indexBytes); + // The metadata fields are a few dozen bytes each, below WarpCopy's vector + // width, which sent them through its byte-at-a-time tail; copy them a word + // per lane instead. + const uint8_t* meta = record + hiddenBytes; + asyncll::LaneCopyWords(outIndices + destTokId * indexBytes, meta, indexBytes); if (args.weightsBuf) - core::WarpCopy( - args.reg(args.offDispatchOutWeights)->template GetAs() + - destTokId * weightBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes + indexBytes, weightBytes); + asyncll::LaneCopyWords(outWeights + destTokId * weightBytes, meta + indexBytes, + weightBytes); if (scaleBytes > 0) - core::WarpCopy( - args.reg(args.offOutScales)->template GetAs() + destTokId * scaleBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes + indexBytes + weightBytes, scaleBytes); + asyncll::LaneCopyWords(outScales + destTokId * scaleBytes, meta + indexBytes + weightBytes, + scaleBytes); if (laneId == 0) { // Where this token came from, so combine's send leg can put the result // back in the slot it arrived in. - args.dispReceiverIdxMap[destTokId] = SendBufSlotOffset(config, destPe, tokenId); + args.dispReceiverIdxMap[destTokId] = SendBufSlotOffset(config, srcPe, tokenId); args.reg(args.offDispTokIdToSrcTokId)->template GetAs()[destTokId] = - reinterpret_cast(stagingPtr + tokenId * xferBytes + hiddenBytes + - indexBytes + weightBytes + scaleBytes)[0]; + reinterpret_cast(meta + indexBytes + weightBytes + scaleBytes)[0]; } } } diff --git a/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp b/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp index 9f25473e73..6ab6c0d6e8 100644 --- a/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp +++ b/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp @@ -183,9 +183,10 @@ mori::jit::v2::LaunchGeometry EpAsyncLLGeometry(const EpInterNodeCfg& cfg, EpAsy const int worldSize = cfg.kernelCfg.worldSize; switch (kind) { - // The copy passes fan out over the whole device. Aligned to worldSize because - // DispatchRecvCopy partitions its grid per source rank; the other two do not - // need it but share the number so a profile lines the three up. + // The copy passes fan out over the whole device. None of them needs the + // worldSize alignment any more -- DispatchRecvCopy used to split its grid per + // source rank and now walks all received tokens -- but they keep it so their + // launch shape, and a profile lining the passes up, stay as they were. case EpAsyncLLKernel::DispatchSlotAssign: case EpAsyncLLKernel::DispatchSendCopy: // The fused LSA copy belongs here and not with the transfers: it walks token