diff --git a/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp b/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp index 0cd0882bc2..e580b2dd83 100644 --- a/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp +++ b/include/mori/ops/dispatch_combine_v2/ep_asyncll_spec.hpp @@ -84,10 +84,10 @@ std::string EpAsyncLLRenderSource(const EpInterNodeCfg& cfg, EpAsyncLLKernel kin mori::jit::v2::LaunchGeometry EpAsyncLLGeometry(const EpInterNodeCfg& cfg, EpAsyncLLKernel kind); // Grids that divide the work per destination rank must be an exact multiple of -// worldSize: DispatchRecvCopy derives `blocksPerPe = gridDim.x / worldSize` and -// then `destPe = blockIdx.x / blocksPerPe`, so a remainder leaves the tail blocks -// decoding to a PE that does not exist. v1 aligns the same way at its launch -// site (`mp - mp % worldSize`). +// worldSize: the LSA transfer passes derive `destPe = globalWarpId / +// (globalWarpNum / worldSize)`, so a remainder leaves the tail warps decoding to +// a PE that does not exist. v1 aligns the same way at its launch site +// (`mp - mp % worldSize`). constexpr int EpAsyncLLAlignGrid(int grid, int worldSize) { const int aligned = grid - (grid % worldSize); return aligned > 0 ? aligned : worldSize; diff --git a/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp b/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp index aa6f795b5e..5c0a20c31e 100644 --- a/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp +++ b/src/ops/dispatch_combine_v2/ep_asyncll_kernel.hpp @@ -180,6 +180,21 @@ __device__ __forceinline__ void PostPayloadAndNotify( } } +// Copy a short field one 4 B word per lane, falling back to one byte per lane +// unless the length and both ends are 4 B aligned. At the shapes in use they are +// (hidden and scale rows are multiples of 4), but the record stride is the sum +// of every field, so an odd scale row would misalign every record after it. +__device__ __forceinline__ void LaneCopyWords(uint8_t* dst, const uint8_t* src, size_t bytes) { + const int lane = threadIdx.x & (warpSize - 1); + if (((bytes | reinterpret_cast(dst) | reinterpret_cast(src)) & 3) == 0) { + for (size_t b = static_cast(lane) * 4; b < bytes; + b += static_cast(warpSize) * 4) + *reinterpret_cast(dst + b) = *reinterpret_cast(src + b); + } else { + for (size_t b = lane; b < bytes; b += warpSize) dst[b] = src[b]; + } +} + } // namespace asyncll /* ---------------------------------------------------------------------------------------------- */ @@ -188,52 +203,99 @@ __device__ __forceinline__ void PostPayloadAndNotify( // Routing only: no payload moves here. Groups of numExpertPerToken lanes inside a // warp cooperate on one token, each lane holding its own expert's destination, so // the deduplication is a register shuffle rather than a reload. +// +// Slots are claimed per BLOCK, not per entry. Every surviving entry needs the +// next slot of its destination PE, and there are only worldSize counters: one +// atomicAdd per entry put ~18k atomics on 8 addresses at EP8, top-6, 4096 tokens +// and the pass took ~58 us for a few hundred KB of routing. Here each warp claims +// its run inside the block (one LDS atomic per destination PE it touches), then +// one thread per PE claims the block's whole run from the global counter: ~6.5 us +// at that shape. The block walks the tokens in lockstep so every __syncthreads +// is reached by all of its threads. +// +// Which slot an entry gets was never deterministic (it was atomic order before), +// and nothing downstream depends on it: RecvCopy compacts each PE's slots in slot +// order and records where each token came from. template __device__ void EpDispatchAsyncLLSlotAssign_body(EpDispatchCombineArgs args) { DEF_COMMON_VARS; + static_assert(kConfig.worldSize <= 64, + "slot claims are tracked in one 64-entry LDS array per destination PE"); constexpr int numEpt = kConfig.numExpertPerToken; const int tokensPerWarp = warpSize / numEpt; + const int tokensPerBlock = warpNum * tokensPerWarp; const int expertIdx = laneId % numEpt; const int inWarpTokIdx = laneId / numEpt; const int baseLane = inWarpTokIdx * numEpt; + const unsigned long long lanesBelow = (1ull << laneId) - 1ull; + + __shared__ index_t blockClaims[64]; + __shared__ index_t blockBase[64]; + + for (int blockTokBase = blockId * tokensPerBlock; blockTokBase < args.curRankNumToken; + blockTokBase += blockNum * tokensPerBlock) { + if (threadIdx.x < kConfig.worldSize) blockClaims[threadIdx.x] = 0; + __syncthreads(); - for (int warpTokBase = globalWarpId * tokensPerWarp; warpTokBase < args.curRankNumToken; - warpTokBase += globalWarpNum * tokensPerWarp) { - const int tokenId = warpTokBase + inWarpTokIdx; + const int tokenId = blockTokBase + warpId * tokensPerWarp + inWarpTokIdx; // warpSize need not be a multiple of numEpt (64 % 6 == 4 at top-k 6): the // trailing lanes alias the next warp's first token, so one entry would get two // slots and the orphan -- never written by SendCopy -- is read as a token by // the receiver. The test at top-k 6/9/10/12 exists for this. // - // Dropping a whole group keeps the shuffle below warp-coherent: inWarpTokIdx - // is group-uniform, so a group is either entirely in or entirely out. - if ((inWarpTokIdx >= tokensPerWarp) || (tokenId >= args.curRankNumToken)) continue; - + // Those lanes and the ones past the last token stay in the loop as inactive + // rather than continuing: the ballots and shuffles below, and the block + // barriers, need every lane. + const bool active = (inWarpTokIdx < tokensPerWarp) && (tokenId < args.curRankNumToken); const int i = tokenId * numEpt + expertIdx; - const index_t destExpert = args.tokenIndices[i]; // A negative expert id is the caller's "drop this slot" sentinel. + const index_t destExpert = active ? args.tokenIndices[i] : static_cast(-1); const bool dropped = (destExpert < 0); const index_t destPe = dropped ? 0 : (destExpert / config.numExpertPerRank); // Deduplicate by shuffling the earlier lanes' destinations out of their - // registers. Every lane runs the same trip count so __shfl stays uniform -- - // an early exit here is what makes it read an inactive lane. + // registers. inWarpTokIdx is group-uniform, so a group is entirely active or + // entirely not, and an inactive group's result is discarded. The mask keeps + // the trailing lanes' source lane inside the wave. bool isDuplicate = false; for (int j = 0; j < numEpt - 1; ++j) { - const index_t prevPe = __shfl(destPe, baseLane + j); - const bool prevDropped = __shfl(static_cast(dropped), baseLane + j) != 0; + const int src = (baseLane + j) & (warpSize - 1); + const index_t prevPe = __shfl(destPe, src); + const bool prevDropped = __shfl(static_cast(dropped), src) != 0; if ((j < expertIdx) && !prevDropped) isDuplicate = isDuplicate || (prevPe == destPe); } // An out-of-range expert id (an EPLB physical id past worldSize*numExpertPerRank) // would index destPeTokenCounter and SendBufSlotOffset out of bounds. Folded - // into the same null branch as the duplicate rather than skipped early, again - // to keep the shuffle above coherent. + // into the same null result as the duplicate rather than skipped early, again + // to keep the shuffles coherent. const bool peOutOfRange = (destPe < 0) || (destPe >= config.worldSize); - if (dropped || isDuplicate || peOutOfRange) { - args.dispDestTokIdMap[i] = NullSendBufSlotOffset(config); - } else { - const index_t destTokId = atomicAdd(args.destPeTokenCounter + destPe, 1); + const bool claims = active && !dropped && !isDuplicate && !peOutOfRange; + + // This lane's offset within the block's run for its PE. + index_t inBlockTokId = 0; +#pragma unroll + for (int pe = 0; pe < kConfig.worldSize; ++pe) { + const unsigned long long lanes = __ballot(claims && (destPe == pe)); + if (lanes) { + const int leader = __ffsll(static_cast(lanes)) - 1; + index_t warpBase = 0; + if (laneId == leader) + warpBase = atomicAdd(&blockClaims[pe], static_cast(__popcll(lanes))); + warpBase = __shfl(warpBase, leader); + if (claims && (destPe == pe)) + inBlockTokId = warpBase + static_cast(__popcll(lanes & lanesBelow)); + } + } + __syncthreads(); + if (threadIdx.x < kConfig.worldSize) { + const index_t n = blockClaims[threadIdx.x]; + blockBase[threadIdx.x] = n ? atomicAdd(args.destPeTokenCounter + threadIdx.x, n) : 0; + } + __syncthreads(); + + if (active) { + const index_t destTokId = claims ? blockBase[destPe] + inBlockTokId : 0; // The counter is cleared by RecvCopy, one per round, so a SECOND // dispatch_send with no recv between it and the first keeps climbing from // where the first left off. Unbounded, SendBufSlotOffset then encodes a @@ -246,10 +308,12 @@ __device__ void EpDispatchAsyncLLSlotAssign_body(EpDispatchCombineArgs args) { // right -- it is the host's call-order check that stops it being reachable // (dispatch_combine_op.py) -- but wrong-and-in-bounds beats corrupting a // peer's buffer, which is the one thing this kernel can still choose. - args.dispDestTokIdMap[i] = (destTokId < config.MaxNumTokensToSendPerRank()) + args.dispDestTokIdMap[i] = (claims && destTokId < config.MaxNumTokensToSendPerRank()) ? SendBufSlotOffset(config, destPe, destTokId) : NullSendBufSlotOffset(config); } + // blockClaims is reset at the top of the next round. + __syncthreads(); } } @@ -610,6 +674,12 @@ __device__ void EpDispatchAsyncLLRecvTransfer_body(EpDispatchCombineArgs args, // Unpack what arrived into the dispatch output, and build the two maps combine // needs. Per-PE output offsets come from a warp prefix sum rather than an atomic, // so every block derives the same layout independently. +// +// The whole grid walks the concatenated output rather than giving each source PE +// a fixed 1/worldSize of the blocks: with a static split the PE that sent the +// most sets the kernel's length while the other groups sit idle, and the PEs a +// rank receives from are rarely even. The owning PE of output row r is the last +// one whose exclusive offset is <= r, read off the prefix sum with one ballot. template __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { DEF_COMMON_VARS; @@ -621,11 +691,6 @@ __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { static_assert(kConfig.worldSize <= 64, "the per-PE offset prefix sum is one wavefront wide; worldSize must fit it"); - const int blocksPerPe = blockNum / npes; - if (blocksPerPe <= 0) return; - const int destPe = blockId / blocksPerPe; - if (destPe >= npes) return; - const uint8_t* recvCounts = args.reg(args.offRecvTokenNum)->template GetAs(); uint64_t myPeTokens = 0; @@ -640,56 +705,63 @@ __device__ void EpDispatchAsyncLLRecvCopy_body(EpDispatchCombineArgs args) { } uint64_t peExclusive = __shfl_up(inclusive, 1); if (laneId == 0) peExclusive = 0; - - const uint64_t peOffset = __shfl(peExclusive, destPe); - const uint64_t recvTokenNum = __shfl(myPeTokens, destPe); const uint64_t totalTokens = __shfl(inclusive, npes - 1); - const uint8_t* stagingPtr = (destPe != myPe) - ? args.reg(args.offDispatchInp)->template GetAs() - : args.reg(args.offStaging)->template GetAs(); - stagingPtr += static_cast(SendBufSlotOffset(config, destPe, 0)) * xferBytes; + const uint8_t* peerBase = args.reg(args.offDispatchInp)->template GetAs(); + // This rank's own tokens were never transferred; they are still in staging. + const uint8_t* selfBase = args.reg(args.offStaging)->template GetAs(); + uint8_t* outTokens = args.reg(args.offDispatchOut)->template GetAs(); + uint8_t* outIndices = args.reg(args.offOutIndices)->template GetAs(); + uint8_t* outWeights = args.reg(args.offDispatchOutWeights)->template GetAs(); + uint8_t* outScales = args.reg(args.offOutScales)->template GetAs(); - const int peWarps = blocksPerPe * warpNum; - const int localWarpId = (blockId % blocksPerPe) * warpNum + warpId; const index_t warpsPerToken = - (recvTokenNum > 0) ? static_cast((peWarps + recvTokenNum - 1) / recvTokenNum) : 1; + (totalTokens > 0) ? static_cast((globalWarpNum + totalTokens - 1) / totalTokens) : 1; const size_t hiddenBytesPerWarp = ((hiddenBytes + warpsPerToken - 1) / warpsPerToken + 15) & ~static_cast(15); - for (int i = localWarpId; i < static_cast(recvTokenNum) * warpsPerToken; i += peWarps) { - const index_t tokenId = i / warpsPerToken; + for (int i = globalWarpId; i < static_cast(totalTokens) * warpsPerToken; + i += globalWarpNum) { + const index_t destTokId = i / warpsPerToken; const index_t inTokenPartId = i % warpsPerToken; - const index_t destTokId = static_cast(peOffset) + tokenId; - + // Lane 0 always qualifies (its exclusive offset is 0), so the mask is never + // empty; a PE that sent nothing shares its offset with the next and is + // skipped by taking the highest qualifying lane. + const unsigned long long owners = + __ballot((laneId < npes) && (peExclusive <= static_cast(destTokId))); + const int srcPe = 63 - __clzll(static_cast(owners)); + const index_t tokenId = destTokId - static_cast(__shfl(peExclusive, srcPe)); + const uint8_t* record = + ((srcPe != myPe) ? peerBase : selfBase) + + (static_cast(SendBufSlotOffset(config, srcPe, 0)) + tokenId) * xferBytes; + + // Four 16 B loads in flight per lane: the source is what SDMA just wrote, so + // it comes from HBM, not cache, and one load per lane left it latency-bound. const size_t hiddenOffset = inTokenPartId * hiddenBytesPerWarp; if (hiddenOffset < hiddenBytes) { const size_t len = min(hiddenBytesPerWarp, hiddenBytes - hiddenOffset); - core::WarpCopy(args.reg(args.offDispatchOut)->template GetAs() + - destTokId * hiddenBytes + hiddenOffset, - stagingPtr + tokenId * xferBytes + hiddenOffset, len); + core::WarpCopy(outTokens + destTokId * hiddenBytes + hiddenOffset, + record + hiddenOffset, len); } if (inTokenPartId == 0) { - core::WarpCopy( - args.reg(args.offOutIndices)->template GetAs() + destTokId * indexBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes, indexBytes); + // The metadata fields are a few dozen bytes each, below WarpCopy's vector + // width, which sent them through its byte-at-a-time tail; copy them a word + // per lane instead. + const uint8_t* meta = record + hiddenBytes; + asyncll::LaneCopyWords(outIndices + destTokId * indexBytes, meta, indexBytes); if (args.weightsBuf) - core::WarpCopy( - args.reg(args.offDispatchOutWeights)->template GetAs() + - destTokId * weightBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes + indexBytes, weightBytes); + asyncll::LaneCopyWords(outWeights + destTokId * weightBytes, meta + indexBytes, + weightBytes); if (scaleBytes > 0) - core::WarpCopy( - args.reg(args.offOutScales)->template GetAs() + destTokId * scaleBytes, - stagingPtr + tokenId * xferBytes + hiddenBytes + indexBytes + weightBytes, scaleBytes); + asyncll::LaneCopyWords(outScales + destTokId * scaleBytes, meta + indexBytes + weightBytes, + scaleBytes); if (laneId == 0) { // Where this token came from, so combine's send leg can put the result // back in the slot it arrived in. - args.dispReceiverIdxMap[destTokId] = SendBufSlotOffset(config, destPe, tokenId); + args.dispReceiverIdxMap[destTokId] = SendBufSlotOffset(config, srcPe, tokenId); args.reg(args.offDispTokIdToSrcTokId)->template GetAs()[destTokId] = - reinterpret_cast(stagingPtr + tokenId * xferBytes + hiddenBytes + - indexBytes + weightBytes + scaleBytes)[0]; + reinterpret_cast(meta + indexBytes + weightBytes + scaleBytes)[0]; } } } diff --git a/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp b/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp index 9f25473e73..6ab6c0d6e8 100644 --- a/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp +++ b/src/ops/dispatch_combine_v2/ep_asyncll_spec.cpp @@ -183,9 +183,10 @@ mori::jit::v2::LaunchGeometry EpAsyncLLGeometry(const EpInterNodeCfg& cfg, EpAsy const int worldSize = cfg.kernelCfg.worldSize; switch (kind) { - // The copy passes fan out over the whole device. Aligned to worldSize because - // DispatchRecvCopy partitions its grid per source rank; the other two do not - // need it but share the number so a profile lines the three up. + // The copy passes fan out over the whole device. None of them needs the + // worldSize alignment any more -- DispatchRecvCopy used to split its grid per + // source rank and now walks all received tokens -- but they keep it so their + // launch shape, and a profile lining the passes up, stay as they were. case EpAsyncLLKernel::DispatchSlotAssign: case EpAsyncLLKernel::DispatchSendCopy: // The fused LSA copy belongs here and not with the transfers: it walks token