From 1c4985ef4a7f52e8175d87954953000181c98fbd Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:38:02 +0000 Subject: [PATCH 01/19] perf(cuda): vectorize the deterministic all-gather copy and right-size its single-block path The all-gather payload copy (single-block fast kernel, fused stage+gather fast kernel, and the multi-block kernel) moved one byte per thread and recomputed the peer with a 64-bit division per byte. Gathers whose output fell in the single-block range (up to 256 KiB) took up to ~1.1 ms on 2 x B200 and ~1.7 ms on 8 x B200, versus ~45-70 us just above it. - copy_rank_ordered: resolve each peer's source/destination once and copy 16-byte vectors when both are aligned, with a byte-wise tail. Still a pure rank-ordered copy; results are byte-identical. - kAllGatherSingleBlockMaxBytes = 64 KiB: a one-block gather copies the whole output, so it only wins below ~64 KiB (measured crossover at 2 and 8 ranks). The reduction fast paths keep kSingleBlockFastPathMaxBytes. - tests/distributed/test_deterministic_all_gather_sizes.py: byte sizes across the threshold, odd tails, misaligned input/output, vs NCCL. - benchmarks/benchmark_deterministic_collectives.py: size sweep for the CUDA deterministic collectives with NCCL references. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 155 ++++++++++++++++++ .../collectives/deterministic_collective.cu | 69 +++++--- .../test_deterministic_all_gather_sizes.py | 102 ++++++++++++ 3 files changed, 303 insertions(+), 23 deletions(-) create mode 100644 benchmarks/benchmark_deterministic_collectives.py create mode 100644 tests/distributed/test_deterministic_all_gather_sizes.py diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py new file mode 100644 index 000000000..7d4a83435 --- /dev/null +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Benchmark the CUDA deterministic collectives across message sizes. + +Example: + + torchrun --standalone --nproc-per-node=8 \ + benchmarks/benchmark_deterministic_collectives.py \ + --output benchmarks/results/deterministic_collectives_b200.json + +``--size-bytes`` is the per-rank input. The default sweep straddles the +single-block fast paths (64 KiB gathered output for ``all_gather``, 256 KiB +for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are +performance references only; they are not a correctness oracle. +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +from pathlib import Path +from typing import Callable, Sequence + +import torch +import torch.distributed as dist + +from rl_engine.distributed.collectives import DeterministicCollective + +_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} +_OPERATIONS = ("all_gather", "all_gather_many", "reduce_scatter", "all_reduce") +_DEFAULT_SIZES = [ + 1024, + 8 * 1024, + 16 * 1024, + 32 * 1024, + 64 * 1024, + 96 * 1024, + 128 * 1024, + 192 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 4 * 1024 * 1024, +] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--size-bytes", type=int, nargs="+", default=_DEFAULT_SIZES) + parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16") + parser.add_argument("--operations", nargs="+", choices=_OPERATIONS, default=_OPERATIONS) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=100) + parser.add_argument("--output", type=Path) + return parser.parse_args(argv) + + +def _median_us(fn: Callable[[], object], warmup: int, iterations: int) -> float: + for _ in range(warmup): + fn() + torch.cuda.synchronize() + samples = [] + for _ in range(iterations): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + fn() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) * 1e3) + return statistics.median(samples) + + +def _slowest(value: float) -> float: + tensor = torch.tensor([value], device="cuda") + dist.all_reduce(tensor, op=dist.ReduceOp.MAX) + return float(tensor.item()) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", rank)) + world = dist.get_world_size() + dtype = _DTYPES[args.dtype] + element = torch.empty((), dtype=dtype).element_size() + capacity = max(args.size_bytes) * world + collective = DeterministicCollective(device=rank, max_size_bytes=capacity) + rows = [] + try: + for size in args.size_bytes: + numel = size // element + x = torch.randn(numel, device="cuda").to(dtype) + gathered = torch.empty(numel * world, device="cuda", dtype=dtype) + reduce_in = torch.randn(numel * world, device="cuda").to(dtype) + cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = { + "all_gather": ( + lambda: collective.all_gather(x, out=gathered), + lambda: dist.all_gather_into_tensor(gathered, x), + ), + "all_gather_many": ( + lambda: collective.all_gather_many((x,)), + lambda: dist.all_gather_into_tensor(gathered, x), + ), + "reduce_scatter": ( + lambda: collective.reduce_scatter(reduce_in), + lambda: dist.reduce_scatter_tensor(x.clone(), reduce_in), + ), + "all_reduce": ( + lambda: collective.all_reduce(x), + lambda: dist.all_reduce(x.clone()), + ), + } + for op in args.operations: + deterministic, nccl = cases[op] + row = { + "operation": op, + "input_bytes_per_rank": numel * element, + "deterministic_us": _slowest( + _median_us(deterministic, args.warmup, args.iterations) + ), + "nccl_us": _slowest(_median_us(nccl, args.warmup, args.iterations)), + } + rows.append(row) + if rank == 0: + print( + f"{op:16s} {row['input_bytes_per_rank']:>9d} B/rank " + f"deterministic {row['deterministic_us']:8.1f} us " + f"nccl {row['nccl_us']:8.1f} us", + flush=True, + ) + finally: + collective.close() + if rank == 0 and args.output is not None: + props = torch.cuda.get_device_properties(rank) + report = { + "kind": "deterministic_collectives_benchmark", + "world_size": world, + "dtype": args.dtype, + "gpu": props.name, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/csrc/cuda/collectives/deterministic_collective.cu b/csrc/cuda/collectives/deterministic_collective.cu index ee562aa64..dd7492efc 100644 --- a/csrc/cuda/collectives/deterministic_collective.cu +++ b/csrc/cuda/collectives/deterministic_collective.cu @@ -35,6 +35,10 @@ constexpr int64_t kSequenceHeaderBytes = 4 * sizeof(uint64_t); // larger transfers, where replacing cudaMemcpyAsync with a one-block copy // would reduce bandwidth and hurt overlap. constexpr int64_t kSingleBlockFastPathMaxBytes = 256 * 1024; +// A one-block all-gather copies the whole gathered output from peer memory, so +// its cost grows with the output while the multi-block path stays near its +// launch floor. On B200 (2 and 8 ranks) the two cross at about a 64 KiB output. +constexpr int64_t kAllGatherSingleBlockMaxBytes = 64 * 1024; // The trace's hot grid=64 collectives are dominated by asymmetric remote // reads. Reduce medium payloads once on logical rank 0 and distribute the // canonical result, while leaving true large transfers on the established @@ -602,6 +606,37 @@ __global__ void deterministic_reduce_scatter_fast_kernel( } } +// The all-gather payload copy: output[peer * input_bytes + i] = peer's byte i, +// in rank order. Each peer's source and destination are resolved once, and +// aligned spans move as 16-byte vectors with a byte-wise tail. Pure data +// movement, so the result is byte-identical to a byte-by-byte copy. +__device__ __forceinline__ void copy_rank_ordered( + const PeerPointers& peers, + int world_size, + uint8_t* output, + int64_t input_bytes, + int64_t thread_index, + int64_t stride) { + for (int peer = 0; peer < world_size; ++peer) { + const auto* source = static_cast(peers.values[peer]); + uint8_t* destination = output + static_cast(peer) * input_bytes; + int64_t vector_bytes = 0; + if (((reinterpret_cast(source) | + reinterpret_cast(destination)) & 15u) == 0u) { + const int64_t vector_count = input_bytes / static_cast(sizeof(uint4)); + const auto* source_vectors = reinterpret_cast(source); + auto* destination_vectors = reinterpret_cast(destination); + for (int64_t index = thread_index; index < vector_count; index += stride) { + destination_vectors[index] = source_vectors[index]; + } + vector_bytes = vector_count * static_cast(sizeof(uint4)); + } + for (int64_t index = vector_bytes + thread_index; index < input_bytes; index += stride) { + destination[index] = source[index]; + } + } +} + __global__ void deterministic_all_gather_fast_kernel( PeerPointers peers, int world_size, @@ -614,12 +649,7 @@ __global__ void deterministic_all_gather_fast_kernel( } __syncthreads(); - const int64_t output_bytes = input_bytes * world_size; - for (int64_t index = threadIdx.x; index < output_bytes; index += blockDim.x) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered(peers, world_size, output, input_bytes, threadIdx.x, blockDim.x); __syncthreads(); if (threadIdx.x == 0) { @@ -978,12 +1008,7 @@ __global__ void deterministic_all_gather_fused_fast_kernel( } __syncthreads(); - const int64_t output_bytes = input_bytes * world_size; - for (int64_t index = threadIdx.x; index < output_bytes; index += blockDim.x) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered(peers, world_size, output, input_bytes, threadIdx.x, blockDim.x); __syncthreads(); if (threadIdx.x == 0) { @@ -1142,15 +1167,13 @@ __global__ void deterministic_all_gather_kernel( uint8_t* output, int64_t input_bytes, int64_t world_size) { - const int64_t output_bytes = input_bytes * world_size; - const int64_t thread_index = - static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - const int64_t stride = static_cast(gridDim.x) * blockDim.x; - for (int64_t index = thread_index; index < output_bytes; index += stride) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered( + peers, + static_cast(world_size), + output, + input_bytes, + static_cast(blockIdx.x) * blockDim.x + threadIdx.x, + static_cast(gridDim.x) * blockDim.x); } class DeterministicCollectiveState { @@ -1650,7 +1673,7 @@ class DeterministicCollectiveState { output_bytes == input_bytes * world_size_, "all-gather output size must contain one input per rank"); - if (output_bytes > kSingleBlockFastPathMaxBytes) { + if (output_bytes > kAllGatherSingleBlockMaxBytes) { stage(input, stream); all_gather(output, stream); return; @@ -1865,7 +1888,7 @@ class DeterministicCollectiveState { "all-gather output must contain one staged input per rank"); const int64_t output_bytes = output.numel() * output.element_size(); - if (staged_fast_path_ && output_bytes <= kSingleBlockFastPathMaxBytes) { + if (staged_fast_path_ && output_bytes <= kAllGatherSingleBlockMaxBytes) { deterministic_all_gather_fast_kernel<<<1, kThreads, 0, stream>>>( peers_, world_size_, diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py new file mode 100644 index 000000000..680880952 --- /dev/null +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Deterministic all-gather is an exact rank-ordered copy at every size. + +The copy has a single-block path (gathered output <= 64 KiB) and a multi-block +path, and moves 16-byte vectors when pointers allow. This sweeps byte counts +across the threshold, odd sizes that leave a byte tail, and misaligned input and +output pointers, and compares every result with NCCL's all-gather. +""" + +from __future__ import annotations + +import os +import socket +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from rl_engine.distributed import create_deterministic_collective + +_WORLD_SIZE = max((n for n in (8, 4, 2) if n <= torch.cuda.device_count()), default=0) +_THRESHOLD = 64 * 1024 # kAllGatherSingleBlockMaxBytes + +pytestmark = [ + pytest.mark.cuda_only, + pytest.mark.skipif( + int(os.environ.get("WORLD_SIZE", "1")) != 1, + reason="this test owns its worker processes; run pytest directly", + ), + pytest.mark.skipif(_WORLD_SIZE < 2, reason="requires at least two visible CUDA GPUs"), +] + + +def _byte_sizes(world_size: int) -> list[int]: + per_rank_threshold = _THRESHOLD // world_size + return sorted( + { + 1, + 15, + 16, + 17, + 4095, + 32 * 1024 + 3, + 96 * 1024, + per_rank_threshold - 1, + per_rank_threshold, + per_rank_threshold + 1, + per_rank_threshold + 16, + 1024 * 1024 + 5, + } + ) + + +def _worker(rank: int, world_size: int, port: int) -> None: + torch.cuda.set_device(rank) + device = torch.device("cuda", rank) + dist.init_process_group( + backend="nccl", + init_method=f"tcp://127.0.0.1:{port}", + rank=rank, + world_size=world_size, + timeout=timedelta(minutes=5), + ) + try: + with create_deterministic_collective( + device=device, max_size_bytes=2 * 1024 * 1024 + ) as collective: + generator = torch.Generator(device="cpu").manual_seed(1234 + rank) + for size in _byte_sizes(world_size): + for input_offset, output_offset in ((0, 0), (1, 0), (0, 3), (5, 7)): + source = torch.randint( + 0, 256, (size + input_offset,), generator=generator, dtype=torch.uint8 + ).to(device) + shard = source[input_offset:] + expected = torch.empty(size * world_size, dtype=torch.uint8, device=device) + dist.all_gather_into_tensor(expected, shard.contiguous()) + + storage = torch.zeros( + size * world_size + output_offset, dtype=torch.uint8, device=device + ) + out = storage[output_offset:] + returned = collective.all_gather(shard, out=out) + assert returned is out + assert torch.equal(out, expected), (size, input_offset, output_offset) + assert torch.count_nonzero(storage[:output_offset]) == 0 + dist.barrier() + finally: + dist.destroy_process_group() + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def test_all_gather_is_exact_across_sizes_and_alignments() -> None: + mp.spawn(_worker, args=(_WORLD_SIZE, _find_free_port()), nprocs=_WORLD_SIZE, join=True) From 0707ac9581f6199df1bda78a29f8538180059b96 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:30:54 +0000 Subject: [PATCH 02/19] bench(cuda): deterministic all-gather before/after results on 2 and 8 x B200 benchmark_deterministic_collectives.py reports for main (43f150f) and this change, all-gather figure (benchmarks/plot_deterministic_collectives.py) and a short report. 2-GPU runs share one machine; 8-GPU runs are on two nodes of the same type. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- benchmarks/plot_deterministic_collectives.py | 102 +++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 benchmarks/plot_deterministic_collectives.py diff --git a/benchmarks/plot_deterministic_collectives.py b/benchmarks/plot_deterministic_collectives.py new file mode 100644 index 000000000..eed5ecb1d --- /dev/null +++ b/benchmarks/plot_deterministic_collectives.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Plot before/after reports of ``benchmark_deterministic_collectives.py``. + + python benchmarks/plot_deterministic_collectives.py \ + --before before_w2.json before_w8.json --after after_w2.json after_w8.json \ + --operation all_gather --output figure.png + +One panel per world size: deterministic time before and after, with NCCL +(from the "after" run) as a reference. Needs only matplotlib. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +BEFORE = "#eb6834" +AFTER = "#2a78d6" +REFERENCE = "#6f6e69" +INK = "#2b2b29" +GRID = "#e4e3dd" +SURFACE = "#fcfcfb" + + +def _rows(path: Path, operation: str) -> tuple[dict, list[dict]]: + report = json.loads(path.read_text()) + rows = [row for row in report["rows"] if row["operation"] == operation] + return report, sorted(rows, key=lambda row: row["input_bytes_per_rank"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--before", type=Path, nargs="+", required=True) + parser.add_argument("--after", type=Path, nargs="+", required=True) + parser.add_argument("--operation", default="all_gather") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if len(args.before) != len(args.after): + raise SystemExit("--before and --after need one report per world size each") + + fig, axes = plt.subplots( + 1, len(args.before), figsize=(5.5 * len(args.before), 3.8), facecolor=SURFACE, squeeze=False + ) + for ax, before_path, after_path in zip(axes[0], args.before, args.after): + before_report, before = _rows(before_path, args.operation) + after_report, after = _rows(after_path, args.operation) + sizes = [row["input_bytes_per_rank"] / 1024 for row in after] + series = ( + ("before", BEFORE, "-", [row["deterministic_us"] for row in before]), + ("after", AFTER, "-", [row["deterministic_us"] for row in after]), + ("NCCL (reference)", REFERENCE, "--", [row["nccl_us"] for row in after]), + ) + for label, color, style, values in series: + ax.plot( + sizes, + values, + style, + color=color, + linewidth=2, + marker="o", + markersize=4, + label=label, + ) + ax.set_xscale("log", base=2) + ax.set_yscale("log") + ax.set_facecolor(SURFACE) + ax.grid(color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + ax.tick_params(colors=REFERENCE, labelsize=8) + ax.set_xlabel("input per rank (KiB)", color=REFERENCE, fontsize=9) + ax.set_ylabel("µs (slowest rank, lower is better)", color=REFERENCE, fontsize=9) + world = after_report["world_size"] + ax.set_title( + f"{args.operation}, {world} x {after_report['gpu']}", loc="left", fontsize=11, color=INK + ) + ax.legend(frameon=False, fontsize=8, labelcolor=INK) + first = json.loads(args.after[0].read_text()) + fig.text( + 0.01, + 0.01, + f"torch {first['torch']} · CUDA {first['cuda']} · {first['dtype']}", + fontsize=7, + color=REFERENCE, + ) + fig.tight_layout(rect=(0, 0.04, 1, 1)) + args.output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(args.output, dpi=150, facecolor=SURFACE) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() From 6bfca957a084dab82aca466cfd3a94b933c21d39 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:16:26 +0000 Subject: [PATCH 03/19] perf(cuda): copy all peers' vectors in one flat loop when aligned The per-peer copy left most threads idle on small per-peer payloads and serialised the peers' remote reads; on 8 x B200 a 64-192 KiB/rank gather was 8-14 us slower than the old byte loop on the multi-block path. With the output, the size and every peer payload 16-byte aligned, one flat vector index over all peers keeps every thread busy; the per-peer copy remains for misaligned pointers. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../collectives/deterministic_collective.cu | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/csrc/cuda/collectives/deterministic_collective.cu b/csrc/cuda/collectives/deterministic_collective.cu index dd7492efc..c9be52269 100644 --- a/csrc/cuda/collectives/deterministic_collective.cu +++ b/csrc/cuda/collectives/deterministic_collective.cu @@ -607,9 +607,11 @@ __global__ void deterministic_reduce_scatter_fast_kernel( } // The all-gather payload copy: output[peer * input_bytes + i] = peer's byte i, -// in rank order. Each peer's source and destination are resolved once, and -// aligned spans move as 16-byte vectors with a byte-wise tail. Pure data -// movement, so the result is byte-identical to a byte-by-byte copy. +// in rank order. When the output, the size and every peer payload are 16-byte +// aligned, one flat index over all peers' 16-byte vectors keeps every thread +// busy across peers; otherwise each peer is copied in turn, with aligned spans +// as vectors and a byte-wise tail. Pure data movement, so the result is +// byte-identical to a byte-by-byte copy. __device__ __forceinline__ void copy_rank_ordered( const PeerPointers& peers, int world_size, @@ -617,6 +619,21 @@ __device__ __forceinline__ void copy_rank_ordered( int64_t input_bytes, int64_t thread_index, int64_t stride) { + uintptr_t alignment = + reinterpret_cast(output) | static_cast(input_bytes); + for (int peer = 0; peer < world_size; ++peer) { + alignment |= reinterpret_cast(peers.values[peer]); + } + if ((alignment & 15u) == 0u) { + const int64_t peer_vectors = input_bytes / static_cast(sizeof(uint4)); + auto* destination = reinterpret_cast(output); + for (int64_t index = thread_index; index < peer_vectors * world_size; index += stride) { + const int64_t peer = index / peer_vectors; + const auto* source = static_cast(peers.values[peer]); + destination[index] = source[index - peer * peer_vectors]; + } + return; + } for (int peer = 0; peer < world_size; ++peer) { const auto* source = static_cast(peers.values[peer]); uint8_t* destination = output + static_cast(peer) * input_bytes; From 7c791704d854818706b31674687a0f2ca555dd1b Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:01:48 +0000 Subject: [PATCH 04/19] fix(bench): isolate NCCL timings and validate paired reports Preallocate NCCL reduction buffers and reset all-reduce inputs before each CUDA event interval. Bind benchmark callbacks to each size's tensors and reject mismatched world sizes or input sweeps before plotting. Add focused regressions, document benchmark helpers, and qualify historical NCCL reduction timings. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 40 +++++-- benchmarks/plot_deterministic_collectives.py | 26 ++++- .../test_deterministic_all_gather_sizes.py | 4 + ...est_benchmark_deterministic_collectives.py | 69 +++++++++++ tests/test_plot_deterministic_collectives.py | 109 ++++++++++++++++++ 5 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 tests/test_benchmark_deterministic_collectives.py create mode 100644 tests/test_plot_deterministic_collectives.py diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index 7d4a83435..528d404a1 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -21,6 +21,7 @@ import json import os import statistics +from functools import partial from pathlib import Path from typing import Callable, Sequence @@ -48,6 +49,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the message-size sweep and distributed benchmark options.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--size-bytes", type=int, nargs="+", default=_DEFAULT_SIZES) parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16") @@ -58,14 +60,25 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _median_us(fn: Callable[[], object], warmup: int, iterations: int) -> float: +def _median_us( + fn: Callable[[], object], + warmup: int, + iterations: int, + *, + prepare: Callable[[], object] | None = None, +) -> float: + """Return median CUDA-event time, excluding preparation before each call.""" for _ in range(warmup): + if prepare is not None: + prepare() fn() torch.cuda.synchronize() samples = [] for _ in range(iterations): start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) + if prepare is not None: + prepare() start.record() fn() end.record() @@ -75,12 +88,14 @@ def _median_us(fn: Callable[[], object], warmup: int, iterations: int) -> float: def _slowest(value: float) -> float: + """Return the largest per-rank timing across the process group.""" tensor = torch.tensor([value], device="cuda") dist.all_reduce(tensor, op=dist.ReduceOp.MAX) return float(tensor.item()) def main(argv: Sequence[str] | None = None) -> None: + """Measure selected collectives on every rank and save the rank-zero report.""" args = parse_args(argv) rank = int(os.environ["LOCAL_RANK"]) torch.cuda.set_device(rank) @@ -97,33 +112,38 @@ def main(argv: Sequence[str] | None = None) -> None: x = torch.randn(numel, device="cuda").to(dtype) gathered = torch.empty(numel * world, device="cuda", dtype=dtype) reduce_in = torch.randn(numel * world, device="cuda").to(dtype) + nccl_scattered = torch.empty_like(x) + nccl_reduced = torch.empty_like(x) cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = { "all_gather": ( - lambda: collective.all_gather(x, out=gathered), - lambda: dist.all_gather_into_tensor(gathered, x), + partial(collective.all_gather, x, out=gathered), + partial(dist.all_gather_into_tensor, gathered, x), ), "all_gather_many": ( - lambda: collective.all_gather_many((x,)), - lambda: dist.all_gather_into_tensor(gathered, x), + partial(collective.all_gather_many, (x,)), + partial(dist.all_gather_into_tensor, gathered, x), ), "reduce_scatter": ( - lambda: collective.reduce_scatter(reduce_in), - lambda: dist.reduce_scatter_tensor(x.clone(), reduce_in), + partial(collective.reduce_scatter, reduce_in), + partial(dist.reduce_scatter_tensor, nccl_scattered, reduce_in), ), "all_reduce": ( - lambda: collective.all_reduce(x), - lambda: dist.all_reduce(x.clone()), + partial(collective.all_reduce, x), + partial(dist.all_reduce, nccl_reduced), ), } for op in args.operations: deterministic, nccl = cases[op] + prepare = partial(nccl_reduced.copy_, x) if op == "all_reduce" else None row = { "operation": op, "input_bytes_per_rank": numel * element, "deterministic_us": _slowest( _median_us(deterministic, args.warmup, args.iterations) ), - "nccl_us": _slowest(_median_us(nccl, args.warmup, args.iterations)), + "nccl_us": _slowest( + _median_us(nccl, args.warmup, args.iterations, prepare=prepare) + ), } rows.append(row) if rank == 0: diff --git a/benchmarks/plot_deterministic_collectives.py b/benchmarks/plot_deterministic_collectives.py index eed5ecb1d..83b23a55a 100644 --- a/benchmarks/plot_deterministic_collectives.py +++ b/benchmarks/plot_deterministic_collectives.py @@ -31,12 +31,14 @@ def _rows(path: Path, operation: str) -> tuple[dict, list[dict]]: + """Load a report and sort the selected operation's rows by input bytes per rank.""" report = json.loads(path.read_text()) rows = [row for row in report["rows"] if row["operation"] == operation] return report, sorted(rows, key=lambda row: row["input_bytes_per_rank"]) def main() -> None: + """Validate paired benchmark reports and save their latency comparison plot.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--before", type=Path, nargs="+", required=True) parser.add_argument("--after", type=Path, nargs="+", required=True) @@ -46,12 +48,28 @@ def main() -> None: if len(args.before) != len(args.after): raise SystemExit("--before and --after need one report per world size each") + reports = [] + for before_path, after_path in zip(args.before, args.after, strict=True): + before_report, before = _rows(before_path, args.operation) + after_report, after = _rows(after_path, args.operation) + if before_report["world_size"] != after_report["world_size"]: + raise SystemExit( + f"{before_path} and {after_path} have different world sizes: " + f"{before_report['world_size']} != {after_report['world_size']}" + ) + before_sizes = [row["input_bytes_per_rank"] for row in before] + after_sizes = [row["input_bytes_per_rank"] for row in after] + if before_sizes != after_sizes: + raise SystemExit( + f"{before_path} and {after_path} have different input sizes for " + f"{args.operation}: {before_sizes} != {after_sizes}" + ) + reports.append((before, after_report, after)) + fig, axes = plt.subplots( 1, len(args.before), figsize=(5.5 * len(args.before), 3.8), facecolor=SURFACE, squeeze=False ) - for ax, before_path, after_path in zip(axes[0], args.before, args.after): - before_report, before = _rows(before_path, args.operation) - after_report, after = _rows(after_path, args.operation) + for ax, (before, after_report, after) in zip(axes[0], reports, strict=True): sizes = [row["input_bytes_per_rank"] / 1024 for row in after] series = ( ("before", BEFORE, "-", [row["deterministic_us"] for row in before]), @@ -84,7 +102,7 @@ def main() -> None: f"{args.operation}, {world} x {after_report['gpu']}", loc="left", fontsize=11, color=INK ) ax.legend(frameon=False, fontsize=8, labelcolor=INK) - first = json.loads(args.after[0].read_text()) + first = reports[0][1] fig.text( 0.01, 0.01, diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py index 680880952..f89256584 100644 --- a/tests/distributed/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -36,6 +36,7 @@ def _byte_sizes(world_size: int) -> list[int]: + """Return per-rank byte counts covering copy boundaries and vector tails.""" per_rank_threshold = _THRESHOLD // world_size return sorted( { @@ -56,6 +57,7 @@ def _byte_sizes(world_size: int) -> list[int]: def _worker(rank: int, world_size: int, port: int) -> None: + """Compare rank-ordered copies with NCCL in an owned process group.""" torch.cuda.set_device(rank) device = torch.device("cuda", rank) dist.init_process_group( @@ -93,10 +95,12 @@ def _worker(rank: int, world_size: int, port: int) -> None: def _find_free_port() -> int: + """Return an available loopback TCP port for the workers' rendezvous.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) return int(sock.getsockname()[1]) def test_all_gather_is_exact_across_sizes_and_alignments() -> None: + """Verify byte-exact output and prefix guards across sizes and offsets.""" mp.spawn(_worker, args=(_WORLD_SIZE, _find_free_port()), nprocs=_WORLD_SIZE, join=True) diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py new file mode 100644 index 000000000..0825a3062 --- /dev/null +++ b/tests/test_benchmark_deterministic_collectives.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""CPU checks for collective benchmark timing boundaries.""" + +import pytest + +from benchmarks import benchmark_deterministic_collectives as benchmark + + +@pytest.mark.parametrize("warmup", [0, 2]) +@pytest.mark.parametrize("with_prepare", [False, True]) +def test_median_us_excludes_preparation(monkeypatch, warmup, with_prepare): + """Reset every call before its timing event without timing the reset.""" + clock_us = 0 + calls = [] + value = 1 + + class Event: + """Model CUDA events on a stream with a deterministic virtual clock.""" + + def __init__(self, *, enable_timing): + """Accept the same timing flag as a CUDA event.""" + assert enable_timing + self.timestamp = None + + def record(self): + """Record the stream's current virtual timestamp.""" + self.timestamp = clock_us + calls.append("event") + + def synchronize(self): + """Represent completion of the recorded event.""" + calls.append("end_sync") + + def elapsed_time(self, end): + """Return elapsed milliseconds, matching the CUDA event API.""" + return (end.timestamp - self.timestamp) / 1e3 + + def prepare(): + """Spend time resetting the mutable collective input.""" + nonlocal clock_us, value + clock_us += 1000 + value = 1 + calls.append("prepare") + + def collective(): + """Model an in-place reduction with seven microseconds of work.""" + nonlocal clock_us, value + if with_prepare: + assert value == 1 + value *= 2 + clock_us += 7 + calls.append("collective") + + monkeypatch.setattr(benchmark.torch.cuda, "Event", Event) + monkeypatch.setattr(benchmark.torch.cuda, "synchronize", lambda: calls.append("sync")) + measured = benchmark._median_us( + collective, warmup, 3, prepare=prepare if with_prepare else None + ) + + assert measured == pytest.approx(7) + preparation = ["prepare"] if with_prepare else [] + assert ( + calls + == (preparation + ["collective"]) * warmup + + ["sync"] + + (preparation + ["event", "collective", "event", "end_sync"]) * 3 + ) diff --git a/tests/test_plot_deterministic_collectives.py b/tests/test_plot_deterministic_collectives.py new file mode 100644 index 000000000..a251664e8 --- /dev/null +++ b/tests/test_plot_deterministic_collectives.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("matplotlib") + +from benchmarks import plot_deterministic_collectives as plotter # noqa: E402 + + +def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) -> None: + """Write a report whose latencies identify each size after sorting.""" + report = { + "world_size": world_size, + "gpu": "test GPU", + "torch": "test", + "cuda": "test", + "dtype": "float32", + "rows": [ + { + "operation": "all_gather", + "input_bytes_per_rank": size, + "deterministic_us": latency + size / 1024, + "nccl_us": size / 1024, + } + for size in sizes + ], + } + path.write_text(json.dumps(report)) + + +@pytest.mark.parametrize( + ("after_world_size", "after_sizes", "message"), + [ + (4, [1024, 2048], "different world sizes"), + (2, [1024, 4096], "different input sizes for all_gather"), + (2, [1024], "different input sizes for all_gather"), + ], +) +def test_mismatched_reports_fail_before_plotting( + tmp_path, monkeypatch, after_world_size, after_sizes, message +): + """Reject differing rank counts or input sweeps before creating a figure.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + _write_report(before_path, 2, [1024, 2048], 10) + _write_report(after_path, after_world_size, after_sizes, 5) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if invalid reports reach the plotting stage.""" + pytest.fail("mismatched reports must be rejected before creating a figure") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises(SystemExit, match=message): + plotter.main() + + +def test_reordered_rows_keep_latencies_aligned(tmp_path, monkeypatch): + """Plot matching sweeps in size order even when source rows are reordered.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + output = tmp_path / "figure.png" + _write_report(before_path, 2, [2048, 1024], 10) + _write_report(after_path, 2, [1024, 2048], 5) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(output), + ], + ) + figure, axes = plotter.plt.subplots(1, 1, squeeze=False) + monkeypatch.setattr(plotter.plt, "subplots", lambda *args, **kwargs: (figure, axes)) + try: + plotter.main() + before, after, reference = axes[0, 0].get_lines() + assert list(before.get_xdata()) == [1, 2] + assert list(before.get_ydata()) == [11, 12] + assert list(after.get_xdata()) == [1, 2] + assert list(after.get_ydata()) == [6, 7] + assert list(reference.get_ydata()) == [1, 2] + assert output.is_file() + finally: + plotter.plt.close(figure) From a290fac40daf0727733bac0d668b31c8efc3b82b Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:14:40 +0000 Subject: [PATCH 05/19] fix(benchmarks): align collective timings and validate report workloads Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 68 ++++---- benchmarks/plot_deterministic_collectives.py | 10 ++ ...est_benchmark_deterministic_collectives.py | 145 ++++++++++++++++-- tests/test_plot_deterministic_collectives.py | 60 +++++++- 4 files changed, 233 insertions(+), 50 deletions(-) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index 528d404a1..4c7fcf190 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -60,31 +60,40 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _median_us( - fn: Callable[[], object], +def _paired_median_us( + deterministic: Callable[[], object], + nccl: Callable[[], object], warmup: int, iterations: int, *, - prepare: Callable[[], object] | None = None, -) -> float: - """Return median CUDA-event time, excluding preparation before each call.""" - for _ in range(warmup): - if prepare is not None: - prepare() - fn() + prepare_nccl: Callable[[], object] | None = None, +) -> tuple[float, float]: + """Time both implementations in alternating order, excluding input resets.""" + functions = (deterministic, nccl) + preparations = (None, prepare_nccl) + for sample in range(warmup): + order = (0, 1) if sample % 2 == 0 else (1, 0) + for index in order: + prepare = preparations[index] + if prepare is not None: + prepare() + functions[index]() torch.cuda.synchronize() - samples = [] - for _ in range(iterations): - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - if prepare is not None: - prepare() - start.record() - fn() - end.record() - end.synchronize() - samples.append(start.elapsed_time(end) * 1e3) - return statistics.median(samples) + samples: tuple[list[float], list[float]] = ([], []) + for sample in range(iterations): + order = (0, 1) if sample % 2 == 0 else (1, 0) + for index in order: + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + prepare = preparations[index] + if prepare is not None: + prepare() + start.record() + functions[index]() + end.record() + end.synchronize() + samples[index].append(start.elapsed_time(end) * 1e3) + return statistics.median(samples[0]), statistics.median(samples[1]) def _slowest(value: float) -> float: @@ -112,6 +121,8 @@ def main(argv: Sequence[str] | None = None) -> None: x = torch.randn(numel, device="cuda").to(dtype) gathered = torch.empty(numel * world, device="cuda", dtype=dtype) reduce_in = torch.randn(numel * world, device="cuda").to(dtype) + deterministic_scattered = torch.empty_like(x) + deterministic_reduced = torch.empty_like(x) nccl_scattered = torch.empty_like(x) nccl_reduced = torch.empty_like(x) cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = { @@ -124,26 +135,25 @@ def main(argv: Sequence[str] | None = None) -> None: partial(dist.all_gather_into_tensor, gathered, x), ), "reduce_scatter": ( - partial(collective.reduce_scatter, reduce_in), + partial(collective.reduce_scatter, reduce_in, out=deterministic_scattered), partial(dist.reduce_scatter_tensor, nccl_scattered, reduce_in), ), "all_reduce": ( - partial(collective.all_reduce, x), + partial(collective.all_reduce, x, out=deterministic_reduced), partial(dist.all_reduce, nccl_reduced), ), } for op in args.operations: deterministic, nccl = cases[op] prepare = partial(nccl_reduced.copy_, x) if op == "all_reduce" else None + deterministic_us, nccl_us = _paired_median_us( + deterministic, nccl, args.warmup, args.iterations, prepare_nccl=prepare + ) row = { "operation": op, "input_bytes_per_rank": numel * element, - "deterministic_us": _slowest( - _median_us(deterministic, args.warmup, args.iterations) - ), - "nccl_us": _slowest( - _median_us(nccl, args.warmup, args.iterations, prepare=prepare) - ), + "deterministic_us": _slowest(deterministic_us), + "nccl_us": _slowest(nccl_us), } rows.append(row) if rank == 0: diff --git a/benchmarks/plot_deterministic_collectives.py b/benchmarks/plot_deterministic_collectives.py index 83b23a55a..9963f7a2c 100644 --- a/benchmarks/plot_deterministic_collectives.py +++ b/benchmarks/plot_deterministic_collectives.py @@ -57,8 +57,18 @@ def main() -> None: f"{before_path} and {after_path} have different world sizes: " f"{before_report['world_size']} != {after_report['world_size']}" ) + for field in ("gpu", "dtype"): + if before_report[field] != after_report[field]: + raise SystemExit( + f"{before_path} and {after_path} have different {field} values: " + f"{before_report[field]!r} != {after_report[field]!r}" + ) before_sizes = [row["input_bytes_per_rank"] for row in before] after_sizes = [row["input_bytes_per_rank"] for row in after] + if not before_sizes or not after_sizes: + raise SystemExit( + f"{before_path} and {after_path} need rows for {args.operation} in both reports" + ) if before_sizes != after_sizes: raise SystemExit( f"{before_path} and {after_path} have different input sizes for " diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py index 0825a3062..e51729c92 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/test_benchmark_deterministic_collectives.py @@ -3,15 +3,18 @@ """CPU checks for collective benchmark timing boundaries.""" +from types import SimpleNamespace + import pytest +import torch from benchmarks import benchmark_deterministic_collectives as benchmark @pytest.mark.parametrize("warmup", [0, 2]) @pytest.mark.parametrize("with_prepare", [False, True]) -def test_median_us_excludes_preparation(monkeypatch, warmup, with_prepare): - """Reset every call before its timing event without timing the reset.""" +def test_paired_median_us_excludes_preparation(monkeypatch, warmup, with_prepare): + """Alternate each sample's order and reset NCCL inputs outside timing events.""" clock_us = 0 calls = [] value = 1 @@ -44,26 +47,138 @@ def prepare(): value = 1 calls.append("prepare") - def collective(): - """Model an in-place reduction with seven microseconds of work.""" + def deterministic(): + """Model an out-of-place reduction with seven microseconds of work.""" + nonlocal clock_us + clock_us += 7 + calls.append("deterministic") + + def nccl(): + """Model an in-place reduction with eleven microseconds of work.""" nonlocal clock_us, value if with_prepare: assert value == 1 value *= 2 - clock_us += 7 - calls.append("collective") + clock_us += 11 + calls.append("nccl") monkeypatch.setattr(benchmark.torch.cuda, "Event", Event) monkeypatch.setattr(benchmark.torch.cuda, "synchronize", lambda: calls.append("sync")) - measured = benchmark._median_us( - collective, warmup, 3, prepare=prepare if with_prepare else None + measured = benchmark._paired_median_us( + deterministic, nccl, warmup, 3, prepare_nccl=prepare if with_prepare else None ) - assert measured == pytest.approx(7) - preparation = ["prepare"] if with_prepare else [] - assert ( - calls - == (preparation + ["collective"]) * warmup - + ["sync"] - + (preparation + ["event", "collective", "event", "end_sync"]) * 3 + assert measured == pytest.approx((7, 11)) + expected = [] + for count, timed in ((warmup, False), (3, True)): + if timed: + expected.append("sync") + for sample in range(count): + order = ("deterministic", "nccl") if sample % 2 == 0 else ("nccl", "deterministic") + for name in order: + if name == "nccl" and with_prepare: + expected.append("prepare") + if timed: + expected.append("event") + expected.append(name) + if timed: + expected.extend(("event", "end_sync")) + assert calls == expected + + +def test_reduction_callbacks_reuse_outputs(monkeypatch): + """Reuse separate reduction buffers while preserving and resetting inputs.""" + outputs = {name: [] for name in ("reduce_scatter", "all_reduce")} + nccl_outputs = {name: [] for name in outputs} + inputs = [] + + class Collective: + """Model deterministic reductions on CPU with caller-provided outputs.""" + + def __init__(self, **kwargs): + """Accept the distributed benchmark configuration.""" + + def all_gather(self, *args, **kwargs): + """Reject gather calls in this reductions-only benchmark.""" + pytest.fail("only reduction callbacks should run") + + all_gather_many = all_gather + + def reduce_scatter(self, value, *, out): + """Write a two-rank reduction into the supplied output buffer.""" + outputs["reduce_scatter"].append(out) + return out.copy_(value.view(2, -1).sum(dim=0)) + + def all_reduce(self, value, *, out): + """Preserve the source while writing a simulated two-rank sum.""" + inputs.append(value.clone()) + assert out is not value + outputs["all_reduce"].append(out) + return out.copy_(value * 2) + + def close(self): + """Release the simulated collective.""" + + def nccl_scatter(out, value): + """Model NCCL's out-of-place reduce-scatter contract.""" + nccl_outputs["reduce_scatter"].append(out) + out.copy_(value.view(2, -1).sum(dim=0)) + + def nccl_reduce(value): + """Require a reset before each simulated in-place NCCL reduction.""" + torch.testing.assert_close(value, inputs[-1]) + nccl_outputs["all_reduce"].append(value) + value.mul_(2) + + def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): + """Invoke the real benchmark callbacks repeatedly without GPU timing.""" + assert (warmup, iterations) == (1, 2) + for _ in range(warmup + iterations): + deterministic() + if prepare_nccl is not None: + prepare_nccl() + nccl() + return 7.0, 11.0 + + cpu_torch = SimpleNamespace( + cuda=SimpleNamespace(set_device=lambda rank: None), + device=torch.device, + empty=lambda *shape, device=None, **kwargs: torch.empty(*shape, **kwargs), + randn=lambda *shape, device=None: torch.randn(*shape), + empty_like=torch.empty_like, + ) + cpu_dist = SimpleNamespace( + init_process_group=lambda *args, **kwargs: None, + get_world_size=lambda: 2, + all_gather_into_tensor=lambda *args: None, + reduce_scatter_tensor=nccl_scatter, + all_reduce=nccl_reduce, + destroy_process_group=lambda: None, ) + monkeypatch.setenv("LOCAL_RANK", "0") + monkeypatch.setattr(benchmark, "torch", cpu_torch) + monkeypatch.setattr(benchmark, "dist", cpu_dist) + monkeypatch.setattr(benchmark, "DeterministicCollective", Collective) + monkeypatch.setattr(benchmark, "_paired_median_us", measure) + monkeypatch.setattr(benchmark, "_slowest", lambda value: value) + benchmark.main( + [ + "--size-bytes", + "32", + "--operations", + "reduce_scatter", + "all_reduce", + "--warmup", + "1", + "--iterations", + "2", + ] + ) + + for name in outputs: + assert len(outputs[name]) == len(nccl_outputs[name]) == 3 + assert all(out is outputs[name][0] for out in outputs[name]) + assert all(out is nccl_outputs[name][0] for out in nccl_outputs[name]) + assert outputs[name][0] is not nccl_outputs[name][0] + torch.testing.assert_close(outputs[name][0], nccl_outputs[name][0]) + assert all(torch.equal(value, inputs[0]) for value in inputs) diff --git a/tests/test_plot_deterministic_collectives.py b/tests/test_plot_deterministic_collectives.py index a251664e8..0cd6857c8 100644 --- a/tests/test_plot_deterministic_collectives.py +++ b/tests/test_plot_deterministic_collectives.py @@ -36,21 +36,26 @@ def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) @pytest.mark.parametrize( - ("after_world_size", "after_sizes", "message"), + ("after_world_size", "after_sizes", "after_metadata", "message"), [ - (4, [1024, 2048], "different world sizes"), - (2, [1024, 4096], "different input sizes for all_gather"), - (2, [1024], "different input sizes for all_gather"), + (4, [1024, 2048], {}, "different world sizes"), + (2, [1024, 4096], {}, "different input sizes for all_gather"), + (2, [1024], {}, "different input sizes for all_gather"), + (2, [1024, 2048], {"gpu": "different GPU"}, "different gpu values"), + (2, [1024, 2048], {"dtype": "bf16"}, "different dtype values"), ], ) def test_mismatched_reports_fail_before_plotting( - tmp_path, monkeypatch, after_world_size, after_sizes, message + tmp_path, monkeypatch, after_world_size, after_sizes, after_metadata, message ): - """Reject differing rank counts or input sweeps before creating a figure.""" + """Reject differing workloads or input sweeps before creating a figure.""" before_path = tmp_path / "before.json" after_path = tmp_path / "after.json" _write_report(before_path, 2, [1024, 2048], 10) _write_report(after_path, after_world_size, after_sizes, 5) + report = json.loads(after_path.read_text()) + report.update(after_metadata) + after_path.write_text(json.dumps(report)) monkeypatch.setattr( sys, "argv", @@ -74,6 +79,49 @@ def unexpected_subplots(*args, **kwargs): plotter.main() +@pytest.mark.parametrize( + ("before_operation", "after_operation"), + [ + ("all_reduce", "all_gather"), + ("all_gather", "all_reduce"), + ("all_reduce", "all_reduce"), + ], +) +def test_missing_operation_fails_before_plotting( + tmp_path, monkeypatch, before_operation, after_operation +): + """Reject a selected operation absent from either or both reports.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + for path, operation in ((before_path, before_operation), (after_path, after_operation)): + _write_report(path, 2, [1024, 2048], 10) + report = json.loads(path.read_text()) + for row in report["rows"]: + row["operation"] = operation + path.write_text(json.dumps(report)) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if an empty selected series reaches the plotting stage.""" + pytest.fail("reports missing the selected operation must fail before plotting") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises(SystemExit, match="need rows for all_gather in both reports"): + plotter.main() + + def test_reordered_rows_keep_latencies_aligned(tmp_path, monkeypatch): """Plot matching sweeps in size order even when source rows are reordered.""" before_path = tmp_path / "before.json" From 6128a72829b54564b5e578226ca887f01c3f4005 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:59:06 +0000 Subject: [PATCH 06/19] fix(collectives): retain timing samples and right-size gather grid Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 34 +++++--- .../collectives/deterministic_collective.cu | 26 ++++-- .../test_deterministic_all_gather_sizes.py | 1 + ...est_benchmark_deterministic_collectives.py | 86 +++++++++++++++---- 4 files changed, 113 insertions(+), 34 deletions(-) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index 4c7fcf190..f995a4850 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -13,6 +13,9 @@ single-block fast paths (64 KiB gathered output for ``all_gather``, 256 KiB for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are performance references only; they are not a correctness oracle. + +JSON timing samples are grouped by global rank, in collection order. The +reported latency is the maximum of those ranks' median sample latencies. """ from __future__ import annotations @@ -60,15 +63,15 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _paired_median_us( +def _paired_samples_us( deterministic: Callable[[], object], nccl: Callable[[], object], warmup: int, iterations: int, *, prepare_nccl: Callable[[], object] | None = None, -) -> tuple[float, float]: - """Time both implementations in alternating order, excluding input resets.""" +) -> tuple[list[float], list[float]]: + """Collect alternating timing samples in microseconds, excluding resets.""" functions = (deterministic, nccl) preparations = (None, prepare_nccl) for sample in range(warmup): @@ -93,14 +96,15 @@ def _paired_median_us( end.record() end.synchronize() samples[index].append(start.elapsed_time(end) * 1e3) - return statistics.median(samples[0]), statistics.median(samples[1]) + return samples -def _slowest(value: float) -> float: - """Return the largest per-rank timing across the process group.""" - tensor = torch.tensor([value], device="cuda") - dist.all_reduce(tensor, op=dist.ReduceOp.MAX) - return float(tensor.item()) +def _gather_samples_us(samples: list[float]) -> list[list[float]]: + """Gather timing samples in global-rank order without reducing their precision.""" + tensor = torch.tensor(samples, device="cuda", dtype=torch.float64) + gathered = [torch.empty_like(tensor) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, tensor) + return [rank_samples.tolist() for rank_samples in gathered] def main(argv: Sequence[str] | None = None) -> None: @@ -146,14 +150,20 @@ def main(argv: Sequence[str] | None = None) -> None: for op in args.operations: deterministic, nccl = cases[op] prepare = partial(nccl_reduced.copy_, x) if op == "all_reduce" else None - deterministic_us, nccl_us = _paired_median_us( + deterministic_samples, nccl_samples = _paired_samples_us( deterministic, nccl, args.warmup, args.iterations, prepare_nccl=prepare ) + deterministic_samples_us = _gather_samples_us(deterministic_samples) + nccl_samples_us = _gather_samples_us(nccl_samples) row = { "operation": op, "input_bytes_per_rank": numel * element, - "deterministic_us": _slowest(deterministic_us), - "nccl_us": _slowest(nccl_us), + "deterministic_us": max( + statistics.median(samples) for samples in deterministic_samples_us + ), + "nccl_us": max(statistics.median(samples) for samples in nccl_samples_us), + "deterministic_samples_us": deterministic_samples_us, + "nccl_samples_us": nccl_samples_us, } rows.append(row) if rank == 0: diff --git a/csrc/cuda/collectives/deterministic_collective.cu b/csrc/cuda/collectives/deterministic_collective.cu index c9be52269..a7ab52c0a 100644 --- a/csrc/cuda/collectives/deterministic_collective.cu +++ b/csrc/cuda/collectives/deterministic_collective.cu @@ -606,6 +606,19 @@ __global__ void deterministic_reduce_scatter_fast_kernel( } } +__host__ __device__ __forceinline__ bool can_vectorize_all_gather_copy( + const PeerPointers& peers, + int world_size, + const void* output, + int64_t input_bytes) { + uintptr_t alignment = + reinterpret_cast(output) | static_cast(input_bytes); + for (int peer = 0; peer < world_size; ++peer) { + alignment |= reinterpret_cast(peers.values[peer]); + } + return (alignment & 15u) == 0u; +} + // The all-gather payload copy: output[peer * input_bytes + i] = peer's byte i, // in rank order. When the output, the size and every peer payload are 16-byte // aligned, one flat index over all peers' 16-byte vectors keeps every thread @@ -619,12 +632,7 @@ __device__ __forceinline__ void copy_rank_ordered( int64_t input_bytes, int64_t thread_index, int64_t stride) { - uintptr_t alignment = - reinterpret_cast(output) | static_cast(input_bytes); - for (int peer = 0; peer < world_size; ++peer) { - alignment |= reinterpret_cast(peers.values[peer]); - } - if ((alignment & 15u) == 0u) { + if (can_vectorize_all_gather_copy(peers, world_size, output, input_bytes)) { const int64_t peer_vectors = input_bytes / static_cast(sizeof(uint4)); auto* destination = reinterpret_cast(output); for (int64_t index = thread_index; index < peer_vectors * world_size; index += stride) { @@ -1923,9 +1931,13 @@ class DeterministicCollectiveState { publish_done(stream); return; } + const int64_t work_items = can_vectorize_all_gather_copy( + peers_, world_size_, output.data_ptr(), staged_bytes_) + ? output_bytes / static_cast(sizeof(uint4)) + : output_bytes; const int blocks = static_cast(std::min( kMaxBlocks, - (output_bytes + kThreads - 1) / kThreads)); + (work_items + kThreads - 1) / kThreads)); deterministic_all_gather_kernel<<>>( peers_, static_cast(output.data_ptr()), diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py index f89256584..3b2016175 100644 --- a/tests/distributed/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -51,6 +51,7 @@ def _byte_sizes(world_size: int) -> list[int]: per_rank_threshold, per_rank_threshold + 1, per_rank_threshold + 16, + 1024 * 1024, 1024 * 1024 + 5, } ) diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py index e51729c92..e0faaf53d 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/test_benchmark_deterministic_collectives.py @@ -3,6 +3,7 @@ """CPU checks for collective benchmark timing boundaries.""" +import json from types import SimpleNamespace import pytest @@ -13,11 +14,15 @@ @pytest.mark.parametrize("warmup", [0, 2]) @pytest.mark.parametrize("with_prepare", [False, True]) -def test_paired_median_us_excludes_preparation(monkeypatch, warmup, with_prepare): +def test_paired_samples_us_excludes_preparation(monkeypatch, warmup, with_prepare): """Alternate each sample's order and reset NCCL inputs outside timing events.""" clock_us = 0 calls = [] value = 1 + deterministic_durations = [17, 5, 9, 4, 12] + nccl_durations = [19, 8, 15, 6, 11] + deterministic_calls = 0 + nccl_calls = 0 class Event: """Model CUDA events on a stream with a deterministic virtual clock.""" @@ -48,27 +53,30 @@ def prepare(): calls.append("prepare") def deterministic(): - """Model an out-of-place reduction with seven microseconds of work.""" - nonlocal clock_us - clock_us += 7 + """Model an out-of-place reduction with varying sample durations.""" + nonlocal clock_us, deterministic_calls + clock_us += deterministic_durations[deterministic_calls] + deterministic_calls += 1 calls.append("deterministic") def nccl(): - """Model an in-place reduction with eleven microseconds of work.""" - nonlocal clock_us, value + """Model an in-place reduction with varying sample durations.""" + nonlocal clock_us, value, nccl_calls if with_prepare: assert value == 1 value *= 2 - clock_us += 11 + clock_us += nccl_durations[nccl_calls] + nccl_calls += 1 calls.append("nccl") monkeypatch.setattr(benchmark.torch.cuda, "Event", Event) monkeypatch.setattr(benchmark.torch.cuda, "synchronize", lambda: calls.append("sync")) - measured = benchmark._paired_median_us( + measured = benchmark._paired_samples_us( deterministic, nccl, warmup, 3, prepare_nccl=prepare if with_prepare else None ) - assert measured == pytest.approx((7, 11)) + assert measured[0] == pytest.approx(deterministic_durations[warmup : warmup + 3]) + assert measured[1] == pytest.approx(nccl_durations[warmup : warmup + 3]) expected = [] for count, timed in ((warmup, False), (3, True)): if timed: @@ -86,8 +94,31 @@ def nccl(): assert calls == expected -def test_reduction_callbacks_reuse_outputs(monkeypatch): - """Reuse separate reduction buffers while preserving and resetting inputs.""" +def test_gather_samples_us_preserves_rank_order_and_precision(monkeypatch): + """Keep each rank's complete ordered samples without float32 rounding.""" + samples_by_rank = [[1.123456789, 3.987654321], [7.876543219, 2.234567891]] + + def gather(outputs, samples): + """Populate gathered tensors in global-rank order.""" + assert samples.dtype == torch.float64 + assert samples.tolist() == samples_by_rank[1] + for output, rank_samples in zip(outputs, samples_by_rank, strict=True): + output.copy_(torch.tensor(rank_samples, dtype=torch.float64)) + + cpu_torch = SimpleNamespace( + tensor=lambda values, *, device, dtype: torch.tensor(values, dtype=dtype), + float64=torch.float64, + empty_like=torch.empty_like, + ) + cpu_dist = SimpleNamespace(get_world_size=lambda: 2, all_gather=gather) + monkeypatch.setattr(benchmark, "torch", cpu_torch) + monkeypatch.setattr(benchmark, "dist", cpu_dist) + + assert benchmark._gather_samples_us(samples_by_rank[1]) == samples_by_rank + + +def test_reduction_callbacks_reuse_outputs_and_report_samples(monkeypatch, tmp_path): + """Reuse reduction buffers and report all ranks with their maximum medians.""" outputs = {name: [] for name in ("reduce_scatter", "all_reduce")} nccl_outputs = {name: [] for name in outputs} inputs = [] @@ -138,14 +169,26 @@ def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): if prepare_nccl is not None: prepare_nccl() nccl() - return 7.0, 11.0 + return [7.0, 19.0], [19.0, 17.0] + + def gather_samples(samples): + """Model different slowest ranks for deterministic and NCCL timings.""" + if samples == [7.0, 19.0]: + return [samples, [20.0, 10.0]] + assert samples == [19.0, 17.0] + return [samples, [21.0, 13.0]] cpu_torch = SimpleNamespace( - cuda=SimpleNamespace(set_device=lambda rank: None), + cuda=SimpleNamespace( + set_device=lambda rank: None, + get_device_properties=lambda rank: SimpleNamespace(name="test-gpu"), + ), device=torch.device, empty=lambda *shape, device=None, **kwargs: torch.empty(*shape, **kwargs), randn=lambda *shape, device=None: torch.randn(*shape), empty_like=torch.empty_like, + __version__="test-torch", + version=SimpleNamespace(cuda="test-cuda"), ) cpu_dist = SimpleNamespace( init_process_group=lambda *args, **kwargs: None, @@ -159,8 +202,9 @@ def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): monkeypatch.setattr(benchmark, "torch", cpu_torch) monkeypatch.setattr(benchmark, "dist", cpu_dist) monkeypatch.setattr(benchmark, "DeterministicCollective", Collective) - monkeypatch.setattr(benchmark, "_paired_median_us", measure) - monkeypatch.setattr(benchmark, "_slowest", lambda value: value) + monkeypatch.setattr(benchmark, "_paired_samples_us", measure) + monkeypatch.setattr(benchmark, "_gather_samples_us", gather_samples) + report_path = tmp_path / "nested" / "timings.json" benchmark.main( [ "--size-bytes", @@ -172,6 +216,8 @@ def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): "1", "--iterations", "2", + "--output", + str(report_path), ] ) @@ -182,3 +228,13 @@ def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): assert outputs[name][0] is not nccl_outputs[name][0] torch.testing.assert_close(outputs[name][0], nccl_outputs[name][0]) assert all(torch.equal(value, inputs[0]) for value in inputs) + + report = json.loads(report_path.read_text()) + assert report["world_size"] == 2 + assert [row["operation"] for row in report["rows"]] == ["reduce_scatter", "all_reduce"] + for row in report["rows"]: + assert row["input_bytes_per_rank"] == 32 + assert row["deterministic_samples_us"] == [[7.0, 19.0], [20.0, 10.0]] + assert row["nccl_samples_us"] == [[19.0, 17.0], [21.0, 13.0]] + assert row["deterministic_us"] == 15.0 + assert row["nccl_us"] == 18.0 From 20e6354581e62bd0f61d7720bd461c57e6f79076 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:24:45 +0000 Subject: [PATCH 07/19] fix(benchmarks): validate collective benchmark inputs and report metadata Reject non-positive iterations and unaligned sizes, allocate the NCCL all_gather_many output inside the timed region like the deterministic path, report the real reduce_scatter input size, and record/compare warmup, iterations, torch and cuda between before/after reports. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 21 ++++++++++++++++--- benchmarks/plot_deterministic_collectives.py | 2 +- ...est_benchmark_deterministic_collectives.py | 13 +++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index f995a4850..c140892bd 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -60,7 +60,16 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--warmup", type=int, default=20) parser.add_argument("--iterations", type=int, default=100) parser.add_argument("--output", type=Path) - return parser.parse_args(argv) + args = parser.parse_args(argv) + if args.iterations <= 0: + parser.error("--iterations must be positive") + element = torch.empty((), dtype=_DTYPES[args.dtype]).element_size() + if any(size <= 0 or size % element != 0 for size in args.size_bytes): + parser.error( + f"--size-bytes values must be positive and divisible by " + f"the {element}-byte dtype element size" + ) + return args def _paired_samples_us( @@ -136,7 +145,9 @@ def main(argv: Sequence[str] | None = None) -> None: ), "all_gather_many": ( partial(collective.all_gather_many, (x,)), - partial(dist.all_gather_into_tensor, gathered, x), + lambda out=gathered, src=x: dist.all_gather_into_tensor( + out.new_empty(out.shape), src + ), ), "reduce_scatter": ( partial(collective.reduce_scatter, reduce_in, out=deterministic_scattered), @@ -157,7 +168,9 @@ def main(argv: Sequence[str] | None = None) -> None: nccl_samples_us = _gather_samples_us(nccl_samples) row = { "operation": op, - "input_bytes_per_rank": numel * element, + "input_bytes_per_rank": ( + reduce_in.numel() * element if op == "reduce_scatter" else numel * element + ), "deterministic_us": max( statistics.median(samples) for samples in deterministic_samples_us ), @@ -182,6 +195,8 @@ def main(argv: Sequence[str] | None = None) -> None: "world_size": world, "dtype": args.dtype, "gpu": props.name, + "warmup": args.warmup, + "iterations": args.iterations, "torch": torch.__version__, "cuda": torch.version.cuda, "rows": rows, diff --git a/benchmarks/plot_deterministic_collectives.py b/benchmarks/plot_deterministic_collectives.py index 9963f7a2c..bca5f739f 100644 --- a/benchmarks/plot_deterministic_collectives.py +++ b/benchmarks/plot_deterministic_collectives.py @@ -57,7 +57,7 @@ def main() -> None: f"{before_path} and {after_path} have different world sizes: " f"{before_report['world_size']} != {after_report['world_size']}" ) - for field in ("gpu", "dtype"): + for field in ("gpu", "dtype", "torch", "cuda", "warmup", "iterations"): if before_report[field] != after_report[field]: raise SystemExit( f"{before_path} and {after_path} have different {field} values: " diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py index e0faaf53d..927b344ae 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/test_benchmark_deterministic_collectives.py @@ -232,9 +232,20 @@ def gather_samples(samples): report = json.loads(report_path.read_text()) assert report["world_size"] == 2 assert [row["operation"] for row in report["rows"]] == ["reduce_scatter", "all_reduce"] + assert (report["warmup"], report["iterations"]) == (1, 2) + assert [row["input_bytes_per_rank"] for row in report["rows"]] == [64, 32] for row in report["rows"]: - assert row["input_bytes_per_rank"] == 32 assert row["deterministic_samples_us"] == [[7.0, 19.0], [20.0, 10.0]] assert row["nccl_samples_us"] == [[19.0, 17.0], [21.0, 13.0]] assert row["deterministic_us"] == 15.0 assert row["nccl_us"] == 18.0 + + +@pytest.mark.parametrize( + "argv", + [["--iterations", "0"], ["--size-bytes", "0"], ["--size-bytes", "33"]], +) +def test_parse_args_rejects_invalid_workloads(argv): + """Reject workloads that would produce no samples or truncated tensors.""" + with pytest.raises(SystemExit): + benchmark.parse_args(argv) From f7f0ff95d1c19f6b1946881c62e7ae32d617b7b2 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:24:46 +0000 Subject: [PATCH 08/19] test(collectives): guard all-gather output prefix with a nonzero sentinel Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- tests/distributed/test_deterministic_all_gather_sizes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py index 3b2016175..bfa099a06 100644 --- a/tests/distributed/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -82,14 +82,17 @@ def _worker(rank: int, world_size: int, port: int) -> None: expected = torch.empty(size * world_size, dtype=torch.uint8, device=device) dist.all_gather_into_tensor(expected, shard.contiguous()) - storage = torch.zeros( - size * world_size + output_offset, dtype=torch.uint8, device=device + storage = torch.full( + (size * world_size + output_offset,), + 0xA5, + dtype=torch.uint8, + device=device, ) out = storage[output_offset:] returned = collective.all_gather(shard, out=out) assert returned is out assert torch.equal(out, expected), (size, input_offset, output_offset) - assert torch.count_nonzero(storage[:output_offset]) == 0 + assert torch.all(storage[:output_offset] == 0xA5) dist.barrier() finally: dist.destroy_process_group() From af60420ea233fdd4c3e736a564689d12d62ae507 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:58:44 +0000 Subject: [PATCH 09/19] docs: document the functions this change touches Add docstrings and doc comments to the Python and CUDA functions added or changed here. Docstring and comment lines only; no code changes. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../collectives/deterministic_collective.cu | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/csrc/cuda/collectives/deterministic_collective.cu b/csrc/cuda/collectives/deterministic_collective.cu index a7ab52c0a..e675ea2a8 100644 --- a/csrc/cuda/collectives/deterministic_collective.cu +++ b/csrc/cuda/collectives/deterministic_collective.cu @@ -606,6 +606,9 @@ __global__ void deterministic_reduce_scatter_fast_kernel( } } +// True when the output pointer, the per-rank byte count and every peer payload pointer +// are all multiples of 16 bytes. Shared by the host grid sizing in all_gather() and the +// device copy in copy_rank_ordered(), so both agree on the flat uint4 path over all peers. __host__ __device__ __forceinline__ bool can_vectorize_all_gather_copy( const PeerPointers& peers, int world_size, @@ -662,6 +665,9 @@ __device__ __forceinline__ void copy_rank_ordered( } } +// One-block all-gather for fast-staged inputs: thread 0 waits until every peer's stage +// sequence reaches this rank's, the block copies all payloads into output in rank order, +// then thread 0 publishes this rank's done sequence. __global__ void deterministic_all_gather_fast_kernel( PeerPointers peers, int world_size, @@ -985,6 +991,9 @@ void launch_all_reduce_fused_fast( } } +// One-block stage + gather: wait for every peer's done sequence, copy input into this +// rank's IPC payload and advance its stage sequence, wait for every peer to stage, copy +// all payloads into output in rank order, then publish this rank's done sequence. __global__ void deterministic_all_gather_fused_fast_kernel( PeerPointers peers, int world_size, @@ -1042,6 +1051,7 @@ __global__ void deterministic_all_gather_fused_fast_kernel( } } +// Launch deterministic_all_gather_fused_fast_kernel as one block of kThreads on stream. void launch_all_gather_fused_fast( const PeerPointers& peers, uint64_t* local_stage_sequence, @@ -1187,6 +1197,9 @@ void launch_reduce_scatter_fast( } } +// Multi-block rank-ordered all-gather copy over a grid-stride loop. Does no +// synchronization: the caller waits for every peer's stage before the launch and +// publishes the done sequence after it. __global__ void deterministic_all_gather_kernel( PeerPointers peers, uint8_t* output, @@ -1681,6 +1694,9 @@ class DeterministicCollectiveState { all_reduce(output, stream, /*allow_owner_path=*/false); } + // Stage input and gather every rank's copy into output in rank order. Outputs up to + // kAllGatherSingleBlockMaxBytes use the one-launch fused kernel; larger outputs fall + // back to stage() followed by all_gather(). void all_gather_fused( torch::Tensor& input, torch::Tensor& output, @@ -1717,6 +1733,9 @@ class DeterministicCollectiveState { AT_CUDA_CHECK(cudaGetLastError()); } + // Gather several tensors under one stage/done sequence: inputs are packed into this + // rank's IPC payload at 16-byte-aligned offsets, then each output receives every rank's + // copy in rank order through cudaMemcpyAsync. void all_gather_many( const std::vector& inputs, const std::vector& outputs, @@ -1901,6 +1920,10 @@ class DeterministicCollectiveState { publish_done(stream); } + // Gather every rank's staged input into output in rank order. Fast-staged inputs with + // output <= kAllGatherSingleBlockMaxBytes use the one-block kernel; otherwise the grid + // is sized from 16-byte vectors when the copy can vectorize, else from bytes, capped at + // kMaxBlocks. void all_gather(torch::Tensor& output, cudaStream_t stream) { check_tensor(output, "output"); TORCH_CHECK(has_staged_input_, "stage() must be called before all_gather()"); @@ -2142,6 +2165,7 @@ void deterministic_collective_all_reduce_fused( state_from_handle(handle)->all_reduce_fused(input, output, stream); } +// Run the fused stage + all-gather for handle on the current stream of input's device. void deterministic_collective_all_gather_fused( int64_t handle, torch::Tensor& input, @@ -2151,6 +2175,7 @@ void deterministic_collective_all_gather_fused( state_from_handle(handle)->all_gather_fused(input, output, stream); } +// Run all_gather_many for handle on the current stream of the first input's device. void deterministic_collective_all_gather_many( int64_t handle, std::vector inputs, @@ -2167,6 +2192,8 @@ void deterministic_collective_reduce_scatter(int64_t handle, torch::Tensor& outp state_from_handle(handle)->reduce_scatter(output, stream); } +// Gather every rank's previously staged input into output on the current stream of +// output's device. void deterministic_collective_all_gather(int64_t handle, torch::Tensor& output) { const c10::cuda::CUDAGuard device_guard(output.device()); auto stream = c10::cuda::getCurrentCUDAStream().stream(); From 9a6192bb86c21b807fa8c9b7f0c9bc06734bb10b Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:59:59 +0000 Subject: [PATCH 10/19] test(collectives): guard the bytes after the all-gather output Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- tests/distributed/test_deterministic_all_gather_sizes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py index bfa099a06..73275bf78 100644 --- a/tests/distributed/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -82,17 +82,19 @@ def _worker(rank: int, world_size: int, port: int) -> None: expected = torch.empty(size * world_size, dtype=torch.uint8, device=device) dist.all_gather_into_tensor(expected, shard.contiguous()) + tail_guard = 16 storage = torch.full( - (size * world_size + output_offset,), + (size * world_size + output_offset + tail_guard,), 0xA5, dtype=torch.uint8, device=device, ) - out = storage[output_offset:] + out = storage[output_offset : output_offset + size * world_size] returned = collective.all_gather(shard, out=out) assert returned is out assert torch.equal(out, expected), (size, input_offset, output_offset) assert torch.all(storage[:output_offset] == 0xA5) + assert torch.all(storage[output_offset + size * world_size :] == 0xA5) dist.barrier() finally: dist.destroy_process_group() From 2c97d5f92af9cf450600cc1b4d1c83ea68226f5c Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:32:31 +0000 Subject: [PATCH 11/19] bench(collectives): report from the global rank and place devices by the local rank Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- benchmarks/benchmark_deterministic_collectives.py | 11 ++++++----- tests/test_benchmark_deterministic_collectives.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index c140892bd..852e13db1 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -119,14 +119,15 @@ def _gather_samples_us(samples: list[float]) -> list[list[float]]: def main(argv: Sequence[str] | None = None) -> None: """Measure selected collectives on every rank and save the rank-zero report.""" args = parse_args(argv) - rank = int(os.environ["LOCAL_RANK"]) - torch.cuda.set_device(rank) - dist.init_process_group("nccl", device_id=torch.device("cuda", rank)) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", local_rank)) + rank = dist.get_rank() world = dist.get_world_size() dtype = _DTYPES[args.dtype] element = torch.empty((), dtype=dtype).element_size() capacity = max(args.size_bytes) * world - collective = DeterministicCollective(device=rank, max_size_bytes=capacity) + collective = DeterministicCollective(device=local_rank, max_size_bytes=capacity) rows = [] try: for size in args.size_bytes: @@ -189,7 +190,7 @@ def main(argv: Sequence[str] | None = None) -> None: finally: collective.close() if rank == 0 and args.output is not None: - props = torch.cuda.get_device_properties(rank) + props = torch.cuda.get_device_properties(local_rank) report = { "kind": "deterministic_collectives_benchmark", "world_size": world, diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py index 927b344ae..a879421b8 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/test_benchmark_deterministic_collectives.py @@ -193,6 +193,7 @@ def gather_samples(samples): cpu_dist = SimpleNamespace( init_process_group=lambda *args, **kwargs: None, get_world_size=lambda: 2, + get_rank=lambda: 0, all_gather_into_tensor=lambda *args: None, reduce_scatter_tensor=nccl_scatter, all_reduce=nccl_reduce, From 746c62b729721deb26dc39bc38525653b82bb05e Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:58:27 +0000 Subject: [PATCH 12/19] test(collectives): repair the plot fixture, reject a negative warmup and cover the staged gather Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- benchmarks/benchmark_deterministic_collectives.py | 2 ++ .../test_deterministic_all_gather_sizes.py | 11 +++++++++++ tests/test_benchmark_deterministic_collectives.py | 2 +- tests/test_plot_deterministic_collectives.py | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/benchmark_deterministic_collectives.py index 852e13db1..ed34bb1ee 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/benchmark_deterministic_collectives.py @@ -63,6 +63,8 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: args = parser.parse_args(argv) if args.iterations <= 0: parser.error("--iterations must be positive") + if args.warmup < 0: + parser.error("--warmup must not be negative") element = torch.empty((), dtype=_DTYPES[args.dtype]).element_size() if any(size <= 0 or size % element != 0 for size in args.size_bytes): parser.error( diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/test_deterministic_all_gather_sizes.py index 73275bf78..8f6a7d052 100644 --- a/tests/distributed/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/test_deterministic_all_gather_sizes.py @@ -95,6 +95,17 @@ def _worker(rank: int, world_size: int, port: int) -> None: assert torch.equal(out, expected), (size, input_offset, output_offset) assert torch.all(storage[:output_offset] == 0xA5) assert torch.all(storage[output_offset + size * world_size :] == 0xA5) + + if (input_offset, output_offset) == (0, 0) and size * world_size <= _THRESHOLD: + # The staged one-block kernel is a separate path from the fused entry. + staged_out = torch.full_like(expected, 0xA5) + collective._extension.deterministic_collective_stage( + collective._handle, shard + ) + collective._extension.deterministic_collective_all_gather( + collective._handle, staged_out + ) + assert torch.equal(staged_out, expected), (size, "staged") dist.barrier() finally: dist.destroy_process_group() diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/test_benchmark_deterministic_collectives.py index a879421b8..deb91ff66 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/test_benchmark_deterministic_collectives.py @@ -244,7 +244,7 @@ def gather_samples(samples): @pytest.mark.parametrize( "argv", - [["--iterations", "0"], ["--size-bytes", "0"], ["--size-bytes", "33"]], + [["--iterations", "0"], ["--warmup", "-1"], ["--size-bytes", "0"], ["--size-bytes", "33"]], ) def test_parse_args_rejects_invalid_workloads(argv): """Reject workloads that would produce no samples or truncated tensors.""" diff --git a/tests/test_plot_deterministic_collectives.py b/tests/test_plot_deterministic_collectives.py index 0cd6857c8..7be2f0b5d 100644 --- a/tests/test_plot_deterministic_collectives.py +++ b/tests/test_plot_deterministic_collectives.py @@ -22,6 +22,8 @@ def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) "torch": "test", "cuda": "test", "dtype": "float32", + "warmup": 1, + "iterations": 2, "rows": [ { "operation": "all_gather", From 9333315c7c2a8ac45f73e8e6310d666472927ad5 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:07:53 +0000 Subject: [PATCH 13/19] refactor(collectives): move the collective benchmarks and tests to the #497 layout Move the benchmark and plot scripts to benchmarks/distributed/, their unit tests to tests/benchmarks/ and the all-gather size test to tests/distributed/collectives/, and update the imports and usage commands. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../{ => distributed}/benchmark_deterministic_collectives.py | 2 +- benchmarks/{ => distributed}/plot_deterministic_collectives.py | 2 +- .../test_benchmark_deterministic_collectives.py | 2 +- tests/{ => benchmarks}/test_plot_deterministic_collectives.py | 2 +- .../{ => collectives}/test_deterministic_all_gather_sizes.py | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename benchmarks/{ => distributed}/benchmark_deterministic_collectives.py (99%) rename benchmarks/{ => distributed}/plot_deterministic_collectives.py (98%) rename tests/{ => benchmarks}/test_benchmark_deterministic_collectives.py (99%) rename tests/{ => benchmarks}/test_plot_deterministic_collectives.py (98%) rename tests/distributed/{ => collectives}/test_deterministic_all_gather_sizes.py (100%) diff --git a/benchmarks/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py similarity index 99% rename from benchmarks/benchmark_deterministic_collectives.py rename to benchmarks/distributed/benchmark_deterministic_collectives.py index ed34bb1ee..4f20a76ea 100644 --- a/benchmarks/benchmark_deterministic_collectives.py +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -6,7 +6,7 @@ Example: torchrun --standalone --nproc-per-node=8 \ - benchmarks/benchmark_deterministic_collectives.py \ + benchmarks/distributed/benchmark_deterministic_collectives.py \ --output benchmarks/results/deterministic_collectives_b200.json ``--size-bytes`` is the per-rank input. The default sweep straddles the diff --git a/benchmarks/plot_deterministic_collectives.py b/benchmarks/distributed/plot_deterministic_collectives.py similarity index 98% rename from benchmarks/plot_deterministic_collectives.py rename to benchmarks/distributed/plot_deterministic_collectives.py index bca5f739f..96078f83a 100644 --- a/benchmarks/plot_deterministic_collectives.py +++ b/benchmarks/distributed/plot_deterministic_collectives.py @@ -3,7 +3,7 @@ """Plot before/after reports of ``benchmark_deterministic_collectives.py``. - python benchmarks/plot_deterministic_collectives.py \ + python benchmarks/distributed/plot_deterministic_collectives.py \ --before before_w2.json before_w8.json --after after_w2.json after_w8.json \ --operation all_gather --output figure.png diff --git a/tests/test_benchmark_deterministic_collectives.py b/tests/benchmarks/test_benchmark_deterministic_collectives.py similarity index 99% rename from tests/test_benchmark_deterministic_collectives.py rename to tests/benchmarks/test_benchmark_deterministic_collectives.py index deb91ff66..5eae1b0d9 100644 --- a/tests/test_benchmark_deterministic_collectives.py +++ b/tests/benchmarks/test_benchmark_deterministic_collectives.py @@ -9,7 +9,7 @@ import pytest import torch -from benchmarks import benchmark_deterministic_collectives as benchmark +from benchmarks.distributed import benchmark_deterministic_collectives as benchmark @pytest.mark.parametrize("warmup", [0, 2]) diff --git a/tests/test_plot_deterministic_collectives.py b/tests/benchmarks/test_plot_deterministic_collectives.py similarity index 98% rename from tests/test_plot_deterministic_collectives.py rename to tests/benchmarks/test_plot_deterministic_collectives.py index 7be2f0b5d..9de2ea925 100644 --- a/tests/test_plot_deterministic_collectives.py +++ b/tests/benchmarks/test_plot_deterministic_collectives.py @@ -11,7 +11,7 @@ pytest.importorskip("matplotlib") -from benchmarks import plot_deterministic_collectives as plotter # noqa: E402 +from benchmarks.distributed import plot_deterministic_collectives as plotter # noqa: E402 def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) -> None: diff --git a/tests/distributed/test_deterministic_all_gather_sizes.py b/tests/distributed/collectives/test_deterministic_all_gather_sizes.py similarity index 100% rename from tests/distributed/test_deterministic_all_gather_sizes.py rename to tests/distributed/collectives/test_deterministic_all_gather_sizes.py From bd9c1fbb3662a5ac84e7478e8dbba7808887f6fa Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:16:32 +0000 Subject: [PATCH 14/19] fix(bench): skip signature validation in the timed deterministic collectives NCCL validates nothing per call, so the deterministic callbacks now pass validate_signature=False. With --warmup 0 an uncached signature exchange no longer lands in the timed samples. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../benchmark_deterministic_collectives.py | 16 ++++++++++++---- .../test_benchmark_deterministic_collectives.py | 6 ++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/benchmarks/distributed/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py index 4f20a76ea..dbcb1f24c 100644 --- a/benchmarks/distributed/benchmark_deterministic_collectives.py +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -141,23 +141,31 @@ def main(argv: Sequence[str] | None = None) -> None: deterministic_reduced = torch.empty_like(x) nccl_scattered = torch.empty_like(x) nccl_reduced = torch.empty_like(x) + # NCCL validates nothing per call, so skip the signature exchange here too; + # otherwise an uncached signature lands in the timed loop when --warmup is 0. + unchecked = {"validate_signature": False} cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = { "all_gather": ( - partial(collective.all_gather, x, out=gathered), + partial(collective.all_gather, x, out=gathered, **unchecked), partial(dist.all_gather_into_tensor, gathered, x), ), "all_gather_many": ( - partial(collective.all_gather_many, (x,)), + partial(collective.all_gather_many, (x,), **unchecked), lambda out=gathered, src=x: dist.all_gather_into_tensor( out.new_empty(out.shape), src ), ), "reduce_scatter": ( - partial(collective.reduce_scatter, reduce_in, out=deterministic_scattered), + partial( + collective.reduce_scatter, + reduce_in, + out=deterministic_scattered, + **unchecked, + ), partial(dist.reduce_scatter_tensor, nccl_scattered, reduce_in), ), "all_reduce": ( - partial(collective.all_reduce, x, out=deterministic_reduced), + partial(collective.all_reduce, x, out=deterministic_reduced, **unchecked), partial(dist.all_reduce, nccl_reduced), ), } diff --git a/tests/benchmarks/test_benchmark_deterministic_collectives.py b/tests/benchmarks/test_benchmark_deterministic_collectives.py index 5eae1b0d9..8878548e3 100644 --- a/tests/benchmarks/test_benchmark_deterministic_collectives.py +++ b/tests/benchmarks/test_benchmark_deterministic_collectives.py @@ -135,13 +135,15 @@ def all_gather(self, *args, **kwargs): all_gather_many = all_gather - def reduce_scatter(self, value, *, out): + def reduce_scatter(self, value, *, out, validate_signature): """Write a two-rank reduction into the supplied output buffer.""" + assert validate_signature is False outputs["reduce_scatter"].append(out) return out.copy_(value.view(2, -1).sum(dim=0)) - def all_reduce(self, value, *, out): + def all_reduce(self, value, *, out, validate_signature): """Preserve the source while writing a simulated two-rank sum.""" + assert validate_signature is False inputs.append(value.clone()) assert out is not value outputs["all_reduce"].append(out) From eb7d18a378101f983a4096797de227214b17f12e Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:07:33 +0000 Subject: [PATCH 15/19] docs(bench): state that --size-bytes is the reduce_scatter output per rank Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../distributed/benchmark_deterministic_collectives.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmarks/distributed/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py index dbcb1f24c..8745106f8 100644 --- a/benchmarks/distributed/benchmark_deterministic_collectives.py +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -9,9 +9,11 @@ benchmarks/distributed/benchmark_deterministic_collectives.py \ --output benchmarks/results/deterministic_collectives_b200.json -``--size-bytes`` is the per-rank input. The default sweep straddles the -single-block fast paths (64 KiB gathered output for ``all_gather``, 256 KiB -for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are +``--size-bytes`` is the per-rank input of ``all_gather`` and ``all_reduce`` and +the per-rank output of ``reduce_scatter``, whose input is world size times that +(the JSON ``input_bytes_per_rank`` field and the plots use the actual input). +The default sweep straddles the single-block fast paths (64 KiB gathered output +for ``all_gather``, 256 KiB for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are performance references only; they are not a correctness oracle. JSON timing samples are grouped by global rank, in collection order. The From c1a5269c30445b2bd63156629312d6de40084ad8 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:07:23 +0000 Subject: [PATCH 16/19] docs(bench): explain why the process group is destroyed after the sweep; name the C++ threshold Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../distributed/benchmark_deterministic_collectives.py | 3 +++ .../collectives/test_deterministic_all_gather_sizes.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/distributed/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py index 8745106f8..b676bc561 100644 --- a/benchmarks/distributed/benchmark_deterministic_collectives.py +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -216,6 +216,9 @@ def main(argv: Sequence[str] | None = None) -> None: } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + "\n") + # Destroy the group on the success path only. On a failing rank, destroy_process_group() + # first flushes this rank's queued NCCL work, so with a peer still inside a collective it + # blocks until the NCCL timeout instead of exiting and letting the launcher stop the peers. dist.destroy_process_group() diff --git a/tests/distributed/collectives/test_deterministic_all_gather_sizes.py b/tests/distributed/collectives/test_deterministic_all_gather_sizes.py index 8f6a7d052..8df644013 100644 --- a/tests/distributed/collectives/test_deterministic_all_gather_sizes.py +++ b/tests/distributed/collectives/test_deterministic_all_gather_sizes.py @@ -3,10 +3,11 @@ """Deterministic all-gather is an exact rank-ordered copy at every size. -The copy has a single-block path (gathered output <= 64 KiB) and a multi-block -path, and moves 16-byte vectors when pointers allow. This sweeps byte counts -across the threshold, odd sizes that leave a byte tail, and misaligned input and -output pointers, and compares every result with NCCL's all-gather. +The copy has a single-block path (gathered output <= 64 KiB, the C++ +``kAllGatherSingleBlockMaxBytes``) and a multi-block path, and moves 16-byte +vectors when pointers allow. This sweeps byte counts across the threshold, odd +sizes that leave a byte tail, and misaligned input and output pointers, and +compares every result with NCCL's all-gather. """ from __future__ import annotations From 28c99752c67444dc8f686fca0ec9ebd1210edb22 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:48:52 +0000 Subject: [PATCH 17/19] docs(bench): state that the all_gather_many row passes a single tensor Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../distributed/benchmark_deterministic_collectives.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/distributed/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py index b676bc561..e4fe0754f 100644 --- a/benchmarks/distributed/benchmark_deterministic_collectives.py +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -13,8 +13,10 @@ the per-rank output of ``reduce_scatter``, whose input is world size times that (the JSON ``input_bytes_per_rank`` field and the plots use the actual input). The default sweep straddles the single-block fast paths (64 KiB gathered output -for ``all_gather``, 256 KiB for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are -performance references only; they are not a correctness oracle. +for ``all_gather``, 256 KiB for the ``reduce_scatter`` input and ``all_reduce``). The +``all_gather_many`` row passes one tensor of ``--size-bytes``, so it is a single-input control for +that call and does not measure the multi-tensor case. NCCL rows are performance references only; +they are not a correctness oracle. JSON timing samples are grouped by global rank, in collection order. The reported latency is the maximum of those ranks' median sample latencies. From 49d4fb3fe3485d9735e0be5568b826d83c1c9055 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:44:00 +0000 Subject: [PATCH 18/19] test(bench): declare matplotlib for dev, check plot environment across worlds, per-op mock samples Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../plot_deterministic_collectives.py | 7 ++++ pyproject.toml | 2 +- ...est_benchmark_deterministic_collectives.py | 31 +++++++++++------ .../test_plot_deterministic_collectives.py | 34 +++++++++++++++++++ 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/benchmarks/distributed/plot_deterministic_collectives.py b/benchmarks/distributed/plot_deterministic_collectives.py index 96078f83a..5c2366975 100644 --- a/benchmarks/distributed/plot_deterministic_collectives.py +++ b/benchmarks/distributed/plot_deterministic_collectives.py @@ -74,6 +74,13 @@ def main() -> None: f"{before_path} and {after_path} have different input sizes for " f"{args.operation}: {before_sizes} != {after_sizes}" ) + if reports: + for field in ("dtype", "torch", "cuda"): + if after_report[field] != reports[0][1][field]: + raise SystemExit( + f"{after_path} and {args.after[0]} have different {field} values: " + f"{after_report[field]!r} != {reports[0][1][field]!r}" + ) reports.append((before, after_report, after)) fig, axes = plt.subplots( diff --git a/pyproject.toml b/pyproject.toml index 045cb471e..244d102f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ cuda = ["flashinfer-python>=0.6.0,<0.7", "nvidia-ml-py"] rocm = ["aiter"] vllm = ["vllm>=0.6.0"] drift-viewer = ["Pillow>=10", "PySide6>=6.6"] -dev = ["pytest", "black", "isort", "ruff", "mypy", "pre-commit"] +dev = ["pytest", "black", "isort", "ruff", "mypy", "pre-commit", "matplotlib"] [tool.setuptools.packages.find] where = ["."] diff --git a/tests/benchmarks/test_benchmark_deterministic_collectives.py b/tests/benchmarks/test_benchmark_deterministic_collectives.py index 8878548e3..8a39bad75 100644 --- a/tests/benchmarks/test_benchmark_deterministic_collectives.py +++ b/tests/benchmarks/test_benchmark_deterministic_collectives.py @@ -163,6 +163,8 @@ def nccl_reduce(value): nccl_outputs["all_reduce"].append(value) value.mul_(2) + measured = [] + def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): """Invoke the real benchmark callbacks repeatedly without GPU timing.""" assert (warmup, iterations) == (1, 2) @@ -171,14 +173,14 @@ def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): if prepare_nccl is not None: prepare_nccl() nccl() - return [7.0, 19.0], [19.0, 17.0] + # Offset the samples per call so each operation's row is distinguishable. + offset = 100.0 * len(measured) + measured.append(offset) + return [7.0 + offset, 19.0 + offset], [19.0 + offset, 17.0 + offset] def gather_samples(samples): - """Model different slowest ranks for deterministic and NCCL timings.""" - if samples == [7.0, 19.0]: - return [samples, [20.0, 10.0]] - assert samples == [19.0, 17.0] - return [samples, [21.0, 13.0]] + """Model a second rank whose samples differ from this rank's.""" + return [samples, [value + 1.0 for value in reversed(samples)]] cpu_torch = SimpleNamespace( cuda=SimpleNamespace( @@ -237,11 +239,18 @@ def gather_samples(samples): assert [row["operation"] for row in report["rows"]] == ["reduce_scatter", "all_reduce"] assert (report["warmup"], report["iterations"]) == (1, 2) assert [row["input_bytes_per_rank"] for row in report["rows"]] == [64, 32] - for row in report["rows"]: - assert row["deterministic_samples_us"] == [[7.0, 19.0], [20.0, 10.0]] - assert row["nccl_samples_us"] == [[19.0, 17.0], [21.0, 13.0]] - assert row["deterministic_us"] == 15.0 - assert row["nccl_us"] == 18.0 + for row, offset in zip(report["rows"], measured, strict=True): + assert row["deterministic_samples_us"] == [ + [7.0 + offset, 19.0 + offset], + [20.0 + offset, 8.0 + offset], + ] + assert row["nccl_samples_us"] == [ + [19.0 + offset, 17.0 + offset], + [18.0 + offset, 20.0 + offset], + ] + assert row["deterministic_us"] == 14.0 + offset + assert row["nccl_us"] == 19.0 + offset + assert measured == [0.0, 100.0] @pytest.mark.parametrize( diff --git a/tests/benchmarks/test_plot_deterministic_collectives.py b/tests/benchmarks/test_plot_deterministic_collectives.py index 9de2ea925..5b4c599ab 100644 --- a/tests/benchmarks/test_plot_deterministic_collectives.py +++ b/tests/benchmarks/test_plot_deterministic_collectives.py @@ -81,6 +81,40 @@ def unexpected_subplots(*args, **kwargs): plotter.main() +@pytest.mark.parametrize("field", ["torch", "cuda", "dtype"]) +def test_environment_must_match_across_world_sizes(tmp_path, monkeypatch, field): + """Reject a multi-panel figure whose reports record different environments.""" + paths = {name: tmp_path / f"{name}.json" for name in ("b2", "a2", "b4", "a4")} + for name, world_size in (("b2", 2), ("a2", 2), ("b4", 4), ("a4", 4)): + _write_report(paths[name], world_size, [1024, 2048], 10 if name[0] == "b" else 5) + report = json.loads(paths["a4"].read_text()) + report[field] = "different" + paths["a4"].write_text(json.dumps(report)) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(paths["b2"]), + str(paths["b4"]), + "--after", + str(paths["a2"]), + str(paths["a4"]), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if reports from different environments reach the plotting stage.""" + pytest.fail("reports from different environments must be rejected") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises(SystemExit, match=f"different {field} values"): + plotter.main() + + @pytest.mark.parametrize( ("before_operation", "after_operation"), [ From 5882864c0e0015c79bbe5c503e0148905c53abb5 Mon Sep 17 00:00:00 2001 From: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:30:47 +0000 Subject: [PATCH 19/19] test(collectives): make the plot tests able to fail on the checks they name The cross-world environment test changed one report of the second pair, so the pair check raised first and the cross-world check was never reached; change both reports and match the cross-world message. The reorder test now uses latency that falls with size and different NCCL series per report, so a wrong sort key or a reference taken from the before report changes the asserted series. Signed-off-by: Wenbo Ji <36562829+fusheng-ji@users.noreply.github.com> --- .../test_plot_deterministic_collectives.py | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/benchmarks/test_plot_deterministic_collectives.py b/tests/benchmarks/test_plot_deterministic_collectives.py index 5b4c599ab..ec3d9486d 100644 --- a/tests/benchmarks/test_plot_deterministic_collectives.py +++ b/tests/benchmarks/test_plot_deterministic_collectives.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import re import sys from pathlib import Path @@ -14,8 +15,19 @@ from benchmarks.distributed import plot_deterministic_collectives as plotter # noqa: E402 -def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) -> None: - """Write a report whose latencies identify each size after sorting.""" +def _write_report( + path: Path, + world_size: int, + sizes: list[int], + latency: float, + slope: float = 1 / 1024, + nccl_offset: float = 0.0, +) -> None: + """Write a report whose latencies identify each size after sorting. + + ``slope`` sets how deterministic latency grows with size (negative: it shrinks, so the + latency order differs from the size order); ``nccl_offset`` tells two reports' NCCL apart. + """ report = { "world_size": world_size, "gpu": "test GPU", @@ -28,8 +40,8 @@ def _write_report(path: Path, world_size: int, sizes: list[int], latency: float) { "operation": "all_gather", "input_bytes_per_rank": size, - "deterministic_us": latency + size / 1024, - "nccl_us": size / 1024, + "deterministic_us": latency + slope * size, + "nccl_us": size / 1024 + nccl_offset, } for size in sizes ], @@ -87,9 +99,12 @@ def test_environment_must_match_across_world_sizes(tmp_path, monkeypatch, field) paths = {name: tmp_path / f"{name}.json" for name in ("b2", "a2", "b4", "a4")} for name, world_size in (("b2", 2), ("a2", 2), ("b4", 4), ("a4", 4)): _write_report(paths[name], world_size, [1024, 2048], 10 if name[0] == "b" else 5) - report = json.loads(paths["a4"].read_text()) - report[field] = "different" - paths["a4"].write_text(json.dumps(report)) + # Change both reports of the second pair: each pair stays consistent (so the pair check + # passes) and only the comparison with the first world size can reject it. + for name in ("b4", "a4"): + report = json.loads(paths[name].read_text()) + report[field] = "different" + paths[name].write_text(json.dumps(report)) monkeypatch.setattr( sys, "argv", @@ -111,7 +126,11 @@ def unexpected_subplots(*args, **kwargs): pytest.fail("reports from different environments must be rejected") monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) - with pytest.raises(SystemExit, match=f"different {field} values"): + with pytest.raises( + SystemExit, + match=f"{re.escape(str(paths['a4']))} and {re.escape(str(paths['a2']))} " + f"have different {field} values", + ): plotter.main() @@ -163,8 +182,10 @@ def test_reordered_rows_keep_latencies_aligned(tmp_path, monkeypatch): before_path = tmp_path / "before.json" after_path = tmp_path / "after.json" output = tmp_path / "figure.png" - _write_report(before_path, 2, [2048, 1024], 10) - _write_report(after_path, 2, [1024, 2048], 5) + # Latency falls with size and the NCCL series differ, so sorting by latency, or taking the + # reference from the before report, would change the asserted series. + _write_report(before_path, 2, [2048, 1024], 100, slope=-1 / 1024, nccl_offset=100) + _write_report(after_path, 2, [1024, 2048], 50, slope=-1 / 1024) monkeypatch.setattr( sys, "argv", @@ -184,9 +205,9 @@ def test_reordered_rows_keep_latencies_aligned(tmp_path, monkeypatch): plotter.main() before, after, reference = axes[0, 0].get_lines() assert list(before.get_xdata()) == [1, 2] - assert list(before.get_ydata()) == [11, 12] + assert list(before.get_ydata()) == [99, 98] assert list(after.get_xdata()) == [1, 2] - assert list(after.get_ydata()) == [6, 7] + assert list(after.get_ydata()) == [49, 48] assert list(reference.get_ydata()) == [1, 2] assert output.is_file() finally: