diff --git a/benchmarks/distributed/benchmark_deterministic_collectives.py b/benchmarks/distributed/benchmark_deterministic_collectives.py new file mode 100644 index 000000000..e4fe0754f --- /dev/null +++ b/benchmarks/distributed/benchmark_deterministic_collectives.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Benchmark the CUDA deterministic collectives across message sizes. + +Example: + + torchrun --standalone --nproc-per-node=8 \ + benchmarks/distributed/benchmark_deterministic_collectives.py \ + --output benchmarks/results/deterministic_collectives_b200.json + +``--size-bytes`` is the per-rank input of ``all_gather`` and ``all_reduce`` and +the per-rank output of ``reduce_scatter``, whose input is world size times that +(the JSON ``input_bytes_per_rank`` field and the plots use the actual input). +The default sweep straddles the single-block fast paths (64 KiB gathered output +for ``all_gather``, 256 KiB for the ``reduce_scatter`` input and ``all_reduce``). The +``all_gather_many`` row passes one tensor of ``--size-bytes``, so it is a single-input control for +that call and does not measure the multi-tensor case. NCCL rows are performance references only; +they are not a correctness oracle. + +JSON timing samples are grouped by global rank, in collection order. The +reported latency is the maximum of those ranks' median sample latencies. +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +from functools import partial +from pathlib import Path +from typing import Callable, Sequence + +import torch +import torch.distributed as dist + +from rl_engine.distributed.collectives import DeterministicCollective + +_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} +_OPERATIONS = ("all_gather", "all_gather_many", "reduce_scatter", "all_reduce") +_DEFAULT_SIZES = [ + 1024, + 8 * 1024, + 16 * 1024, + 32 * 1024, + 64 * 1024, + 96 * 1024, + 128 * 1024, + 192 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 4 * 1024 * 1024, +] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the message-size sweep and distributed benchmark options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--size-bytes", type=int, nargs="+", default=_DEFAULT_SIZES) + parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16") + parser.add_argument("--operations", nargs="+", choices=_OPERATIONS, default=_OPERATIONS) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=100) + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + if args.iterations <= 0: + parser.error("--iterations must be positive") + if args.warmup < 0: + parser.error("--warmup must not be negative") + element = torch.empty((), dtype=_DTYPES[args.dtype]).element_size() + if any(size <= 0 or size % element != 0 for size in args.size_bytes): + parser.error( + f"--size-bytes values must be positive and divisible by " + f"the {element}-byte dtype element size" + ) + return args + + +def _paired_samples_us( + deterministic: Callable[[], object], + nccl: Callable[[], object], + warmup: int, + iterations: int, + *, + prepare_nccl: Callable[[], object] | None = None, +) -> tuple[list[float], list[float]]: + """Collect alternating timing samples in microseconds, excluding resets.""" + functions = (deterministic, nccl) + preparations = (None, prepare_nccl) + for sample in range(warmup): + order = (0, 1) if sample % 2 == 0 else (1, 0) + for index in order: + prepare = preparations[index] + if prepare is not None: + prepare() + functions[index]() + torch.cuda.synchronize() + samples: tuple[list[float], list[float]] = ([], []) + for sample in range(iterations): + order = (0, 1) if sample % 2 == 0 else (1, 0) + for index in order: + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + prepare = preparations[index] + if prepare is not None: + prepare() + start.record() + functions[index]() + end.record() + end.synchronize() + samples[index].append(start.elapsed_time(end) * 1e3) + return samples + + +def _gather_samples_us(samples: list[float]) -> list[list[float]]: + """Gather timing samples in global-rank order without reducing their precision.""" + tensor = torch.tensor(samples, device="cuda", dtype=torch.float64) + gathered = [torch.empty_like(tensor) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, tensor) + return [rank_samples.tolist() for rank_samples in gathered] + + +def main(argv: Sequence[str] | None = None) -> None: + """Measure selected collectives on every rank and save the rank-zero report.""" + args = parse_args(argv) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", local_rank)) + rank = dist.get_rank() + world = dist.get_world_size() + dtype = _DTYPES[args.dtype] + element = torch.empty((), dtype=dtype).element_size() + capacity = max(args.size_bytes) * world + collective = DeterministicCollective(device=local_rank, max_size_bytes=capacity) + rows = [] + try: + for size in args.size_bytes: + numel = size // element + x = torch.randn(numel, device="cuda").to(dtype) + gathered = torch.empty(numel * world, device="cuda", dtype=dtype) + reduce_in = torch.randn(numel * world, device="cuda").to(dtype) + deterministic_scattered = torch.empty_like(x) + deterministic_reduced = torch.empty_like(x) + nccl_scattered = torch.empty_like(x) + nccl_reduced = torch.empty_like(x) + # NCCL validates nothing per call, so skip the signature exchange here too; + # otherwise an uncached signature lands in the timed loop when --warmup is 0. + unchecked = {"validate_signature": False} + cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = { + "all_gather": ( + partial(collective.all_gather, x, out=gathered, **unchecked), + partial(dist.all_gather_into_tensor, gathered, x), + ), + "all_gather_many": ( + partial(collective.all_gather_many, (x,), **unchecked), + lambda out=gathered, src=x: dist.all_gather_into_tensor( + out.new_empty(out.shape), src + ), + ), + "reduce_scatter": ( + partial( + collective.reduce_scatter, + reduce_in, + out=deterministic_scattered, + **unchecked, + ), + partial(dist.reduce_scatter_tensor, nccl_scattered, reduce_in), + ), + "all_reduce": ( + partial(collective.all_reduce, x, out=deterministic_reduced, **unchecked), + partial(dist.all_reduce, nccl_reduced), + ), + } + for op in args.operations: + deterministic, nccl = cases[op] + prepare = partial(nccl_reduced.copy_, x) if op == "all_reduce" else None + deterministic_samples, nccl_samples = _paired_samples_us( + deterministic, nccl, args.warmup, args.iterations, prepare_nccl=prepare + ) + deterministic_samples_us = _gather_samples_us(deterministic_samples) + nccl_samples_us = _gather_samples_us(nccl_samples) + row = { + "operation": op, + "input_bytes_per_rank": ( + reduce_in.numel() * element if op == "reduce_scatter" else numel * element + ), + "deterministic_us": max( + statistics.median(samples) for samples in deterministic_samples_us + ), + "nccl_us": max(statistics.median(samples) for samples in nccl_samples_us), + "deterministic_samples_us": deterministic_samples_us, + "nccl_samples_us": nccl_samples_us, + } + rows.append(row) + if rank == 0: + print( + f"{op:16s} {row['input_bytes_per_rank']:>9d} B/rank " + f"deterministic {row['deterministic_us']:8.1f} us " + f"nccl {row['nccl_us']:8.1f} us", + flush=True, + ) + finally: + collective.close() + if rank == 0 and args.output is not None: + props = torch.cuda.get_device_properties(local_rank) + report = { + "kind": "deterministic_collectives_benchmark", + "world_size": world, + "dtype": args.dtype, + "gpu": props.name, + "warmup": args.warmup, + "iterations": args.iterations, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + # Destroy the group on the success path only. On a failing rank, destroy_process_group() + # first flushes this rank's queued NCCL work, so with a peer still inside a collective it + # blocks until the NCCL timeout instead of exiting and letting the launcher stop the peers. + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/distributed/plot_deterministic_collectives.py b/benchmarks/distributed/plot_deterministic_collectives.py new file mode 100644 index 000000000..5c2366975 --- /dev/null +++ b/benchmarks/distributed/plot_deterministic_collectives.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Plot before/after reports of ``benchmark_deterministic_collectives.py``. + + python benchmarks/distributed/plot_deterministic_collectives.py \ + --before before_w2.json before_w8.json --after after_w2.json after_w8.json \ + --operation all_gather --output figure.png + +One panel per world size: deterministic time before and after, with NCCL +(from the "after" run) as a reference. Needs only matplotlib. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +BEFORE = "#eb6834" +AFTER = "#2a78d6" +REFERENCE = "#6f6e69" +INK = "#2b2b29" +GRID = "#e4e3dd" +SURFACE = "#fcfcfb" + + +def _rows(path: Path, operation: str) -> tuple[dict, list[dict]]: + """Load a report and sort the selected operation's rows by input bytes per rank.""" + report = json.loads(path.read_text()) + rows = [row for row in report["rows"] if row["operation"] == operation] + return report, sorted(rows, key=lambda row: row["input_bytes_per_rank"]) + + +def main() -> None: + """Validate paired benchmark reports and save their latency comparison plot.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--before", type=Path, nargs="+", required=True) + parser.add_argument("--after", type=Path, nargs="+", required=True) + parser.add_argument("--operation", default="all_gather") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if len(args.before) != len(args.after): + raise SystemExit("--before and --after need one report per world size each") + + reports = [] + for before_path, after_path in zip(args.before, args.after, strict=True): + before_report, before = _rows(before_path, args.operation) + after_report, after = _rows(after_path, args.operation) + if before_report["world_size"] != after_report["world_size"]: + raise SystemExit( + f"{before_path} and {after_path} have different world sizes: " + f"{before_report['world_size']} != {after_report['world_size']}" + ) + for field in ("gpu", "dtype", "torch", "cuda", "warmup", "iterations"): + if before_report[field] != after_report[field]: + raise SystemExit( + f"{before_path} and {after_path} have different {field} values: " + f"{before_report[field]!r} != {after_report[field]!r}" + ) + before_sizes = [row["input_bytes_per_rank"] for row in before] + after_sizes = [row["input_bytes_per_rank"] for row in after] + if not before_sizes or not after_sizes: + raise SystemExit( + f"{before_path} and {after_path} need rows for {args.operation} in both reports" + ) + if before_sizes != after_sizes: + raise SystemExit( + f"{before_path} and {after_path} have different input sizes for " + f"{args.operation}: {before_sizes} != {after_sizes}" + ) + if reports: + for field in ("dtype", "torch", "cuda"): + if after_report[field] != reports[0][1][field]: + raise SystemExit( + f"{after_path} and {args.after[0]} have different {field} values: " + f"{after_report[field]!r} != {reports[0][1][field]!r}" + ) + reports.append((before, after_report, after)) + + fig, axes = plt.subplots( + 1, len(args.before), figsize=(5.5 * len(args.before), 3.8), facecolor=SURFACE, squeeze=False + ) + for ax, (before, after_report, after) in zip(axes[0], reports, strict=True): + sizes = [row["input_bytes_per_rank"] / 1024 for row in after] + series = ( + ("before", BEFORE, "-", [row["deterministic_us"] for row in before]), + ("after", AFTER, "-", [row["deterministic_us"] for row in after]), + ("NCCL (reference)", REFERENCE, "--", [row["nccl_us"] for row in after]), + ) + for label, color, style, values in series: + ax.plot( + sizes, + values, + style, + color=color, + linewidth=2, + marker="o", + markersize=4, + label=label, + ) + ax.set_xscale("log", base=2) + ax.set_yscale("log") + ax.set_facecolor(SURFACE) + ax.grid(color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + ax.tick_params(colors=REFERENCE, labelsize=8) + ax.set_xlabel("input per rank (KiB)", color=REFERENCE, fontsize=9) + ax.set_ylabel("µs (slowest rank, lower is better)", color=REFERENCE, fontsize=9) + world = after_report["world_size"] + ax.set_title( + f"{args.operation}, {world} x {after_report['gpu']}", loc="left", fontsize=11, color=INK + ) + ax.legend(frameon=False, fontsize=8, labelcolor=INK) + first = reports[0][1] + fig.text( + 0.01, + 0.01, + f"torch {first['torch']} · CUDA {first['cuda']} · {first['dtype']}", + fontsize=7, + color=REFERENCE, + ) + fig.tight_layout(rect=(0, 0.04, 1, 1)) + args.output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(args.output, dpi=150, facecolor=SURFACE) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/csrc/cuda/collectives/deterministic_collective.cu b/csrc/cuda/collectives/deterministic_collective.cu index ee562aa64..e675ea2a8 100644 --- a/csrc/cuda/collectives/deterministic_collective.cu +++ b/csrc/cuda/collectives/deterministic_collective.cu @@ -35,6 +35,10 @@ constexpr int64_t kSequenceHeaderBytes = 4 * sizeof(uint64_t); // larger transfers, where replacing cudaMemcpyAsync with a one-block copy // would reduce bandwidth and hurt overlap. constexpr int64_t kSingleBlockFastPathMaxBytes = 256 * 1024; +// A one-block all-gather copies the whole gathered output from peer memory, so +// its cost grows with the output while the multi-block path stays near its +// launch floor. On B200 (2 and 8 ranks) the two cross at about a 64 KiB output. +constexpr int64_t kAllGatherSingleBlockMaxBytes = 64 * 1024; // The trace's hot grid=64 collectives are dominated by asymmetric remote // reads. Reduce medium payloads once on logical rank 0 and distribute the // canonical result, while leaving true large transfers on the established @@ -602,6 +606,68 @@ __global__ void deterministic_reduce_scatter_fast_kernel( } } +// True when the output pointer, the per-rank byte count and every peer payload pointer +// are all multiples of 16 bytes. Shared by the host grid sizing in all_gather() and the +// device copy in copy_rank_ordered(), so both agree on the flat uint4 path over all peers. +__host__ __device__ __forceinline__ bool can_vectorize_all_gather_copy( + const PeerPointers& peers, + int world_size, + const void* output, + int64_t input_bytes) { + uintptr_t alignment = + reinterpret_cast(output) | static_cast(input_bytes); + for (int peer = 0; peer < world_size; ++peer) { + alignment |= reinterpret_cast(peers.values[peer]); + } + return (alignment & 15u) == 0u; +} + +// The all-gather payload copy: output[peer * input_bytes + i] = peer's byte i, +// in rank order. When the output, the size and every peer payload are 16-byte +// aligned, one flat index over all peers' 16-byte vectors keeps every thread +// busy across peers; otherwise each peer is copied in turn, with aligned spans +// as vectors and a byte-wise tail. Pure data movement, so the result is +// byte-identical to a byte-by-byte copy. +__device__ __forceinline__ void copy_rank_ordered( + const PeerPointers& peers, + int world_size, + uint8_t* output, + int64_t input_bytes, + int64_t thread_index, + int64_t stride) { + if (can_vectorize_all_gather_copy(peers, world_size, output, input_bytes)) { + const int64_t peer_vectors = input_bytes / static_cast(sizeof(uint4)); + auto* destination = reinterpret_cast(output); + for (int64_t index = thread_index; index < peer_vectors * world_size; index += stride) { + const int64_t peer = index / peer_vectors; + const auto* source = static_cast(peers.values[peer]); + destination[index] = source[index - peer * peer_vectors]; + } + return; + } + for (int peer = 0; peer < world_size; ++peer) { + const auto* source = static_cast(peers.values[peer]); + uint8_t* destination = output + static_cast(peer) * input_bytes; + int64_t vector_bytes = 0; + if (((reinterpret_cast(source) | + reinterpret_cast(destination)) & 15u) == 0u) { + const int64_t vector_count = input_bytes / static_cast(sizeof(uint4)); + const auto* source_vectors = reinterpret_cast(source); + auto* destination_vectors = reinterpret_cast(destination); + for (int64_t index = thread_index; index < vector_count; index += stride) { + destination_vectors[index] = source_vectors[index]; + } + vector_bytes = vector_count * static_cast(sizeof(uint4)); + } + for (int64_t index = vector_bytes + thread_index; index < input_bytes; index += stride) { + destination[index] = source[index]; + } + } +} + +// One-block all-gather for fast-staged inputs: thread 0 waits until every peer's stage +// sequence reaches this rank's, the block copies all payloads into output in rank order, +// then thread 0 publishes this rank's done sequence. __global__ void deterministic_all_gather_fast_kernel( PeerPointers peers, int world_size, @@ -614,12 +680,7 @@ __global__ void deterministic_all_gather_fast_kernel( } __syncthreads(); - const int64_t output_bytes = input_bytes * world_size; - for (int64_t index = threadIdx.x; index < output_bytes; index += blockDim.x) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered(peers, world_size, output, input_bytes, threadIdx.x, blockDim.x); __syncthreads(); if (threadIdx.x == 0) { @@ -930,6 +991,9 @@ void launch_all_reduce_fused_fast( } } +// One-block stage + gather: wait for every peer's done sequence, copy input into this +// rank's IPC payload and advance its stage sequence, wait for every peer to stage, copy +// all payloads into output in rank order, then publish this rank's done sequence. __global__ void deterministic_all_gather_fused_fast_kernel( PeerPointers peers, int world_size, @@ -978,12 +1042,7 @@ __global__ void deterministic_all_gather_fused_fast_kernel( } __syncthreads(); - const int64_t output_bytes = input_bytes * world_size; - for (int64_t index = threadIdx.x; index < output_bytes; index += blockDim.x) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered(peers, world_size, output, input_bytes, threadIdx.x, blockDim.x); __syncthreads(); if (threadIdx.x == 0) { @@ -992,6 +1051,7 @@ __global__ void deterministic_all_gather_fused_fast_kernel( } } +// Launch deterministic_all_gather_fused_fast_kernel as one block of kThreads on stream. void launch_all_gather_fused_fast( const PeerPointers& peers, uint64_t* local_stage_sequence, @@ -1137,20 +1197,21 @@ void launch_reduce_scatter_fast( } } +// Multi-block rank-ordered all-gather copy over a grid-stride loop. Does no +// synchronization: the caller waits for every peer's stage before the launch and +// publishes the done sequence after it. __global__ void deterministic_all_gather_kernel( PeerPointers peers, uint8_t* output, int64_t input_bytes, int64_t world_size) { - const int64_t output_bytes = input_bytes * world_size; - const int64_t thread_index = - static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - const int64_t stride = static_cast(gridDim.x) * blockDim.x; - for (int64_t index = thread_index; index < output_bytes; index += stride) { - const int peer = static_cast(index / input_bytes); - const int64_t peer_offset = index - static_cast(peer) * input_bytes; - output[index] = static_cast(peers.values[peer])[peer_offset]; - } + copy_rank_ordered( + peers, + static_cast(world_size), + output, + input_bytes, + static_cast(blockIdx.x) * blockDim.x + threadIdx.x, + static_cast(gridDim.x) * blockDim.x); } class DeterministicCollectiveState { @@ -1633,6 +1694,9 @@ class DeterministicCollectiveState { all_reduce(output, stream, /*allow_owner_path=*/false); } + // Stage input and gather every rank's copy into output in rank order. Outputs up to + // kAllGatherSingleBlockMaxBytes use the one-launch fused kernel; larger outputs fall + // back to stage() followed by all_gather(). void all_gather_fused( torch::Tensor& input, torch::Tensor& output, @@ -1650,7 +1714,7 @@ class DeterministicCollectiveState { output_bytes == input_bytes * world_size_, "all-gather output size must contain one input per rank"); - if (output_bytes > kSingleBlockFastPathMaxBytes) { + if (output_bytes > kAllGatherSingleBlockMaxBytes) { stage(input, stream); all_gather(output, stream); return; @@ -1669,6 +1733,9 @@ class DeterministicCollectiveState { AT_CUDA_CHECK(cudaGetLastError()); } + // Gather several tensors under one stage/done sequence: inputs are packed into this + // rank's IPC payload at 16-byte-aligned offsets, then each output receives every rank's + // copy in rank order through cudaMemcpyAsync. void all_gather_many( const std::vector& inputs, const std::vector& outputs, @@ -1853,6 +1920,10 @@ class DeterministicCollectiveState { publish_done(stream); } + // Gather every rank's staged input into output in rank order. Fast-staged inputs with + // output <= kAllGatherSingleBlockMaxBytes use the one-block kernel; otherwise the grid + // is sized from 16-byte vectors when the copy can vectorize, else from bytes, capped at + // kMaxBlocks. void all_gather(torch::Tensor& output, cudaStream_t stream) { check_tensor(output, "output"); TORCH_CHECK(has_staged_input_, "stage() must be called before all_gather()"); @@ -1865,7 +1936,7 @@ class DeterministicCollectiveState { "all-gather output must contain one staged input per rank"); const int64_t output_bytes = output.numel() * output.element_size(); - if (staged_fast_path_ && output_bytes <= kSingleBlockFastPathMaxBytes) { + if (staged_fast_path_ && output_bytes <= kAllGatherSingleBlockMaxBytes) { deterministic_all_gather_fast_kernel<<<1, kThreads, 0, stream>>>( peers_, world_size_, @@ -1883,9 +1954,13 @@ class DeterministicCollectiveState { publish_done(stream); return; } + const int64_t work_items = can_vectorize_all_gather_copy( + peers_, world_size_, output.data_ptr(), staged_bytes_) + ? output_bytes / static_cast(sizeof(uint4)) + : output_bytes; const int blocks = static_cast(std::min( kMaxBlocks, - (output_bytes + kThreads - 1) / kThreads)); + (work_items + kThreads - 1) / kThreads)); deterministic_all_gather_kernel<<>>( peers_, static_cast(output.data_ptr()), @@ -2090,6 +2165,7 @@ void deterministic_collective_all_reduce_fused( state_from_handle(handle)->all_reduce_fused(input, output, stream); } +// Run the fused stage + all-gather for handle on the current stream of input's device. void deterministic_collective_all_gather_fused( int64_t handle, torch::Tensor& input, @@ -2099,6 +2175,7 @@ void deterministic_collective_all_gather_fused( state_from_handle(handle)->all_gather_fused(input, output, stream); } +// Run all_gather_many for handle on the current stream of the first input's device. void deterministic_collective_all_gather_many( int64_t handle, std::vector inputs, @@ -2115,6 +2192,8 @@ void deterministic_collective_reduce_scatter(int64_t handle, torch::Tensor& outp state_from_handle(handle)->reduce_scatter(output, stream); } +// Gather every rank's previously staged input into output on the current stream of +// output's device. void deterministic_collective_all_gather(int64_t handle, torch::Tensor& output) { const c10::cuda::CUDAGuard device_guard(output.device()); auto stream = c10::cuda::getCurrentCUDAStream().stream(); diff --git a/pyproject.toml b/pyproject.toml index 045cb471e..244d102f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ cuda = ["flashinfer-python>=0.6.0,<0.7", "nvidia-ml-py"] rocm = ["aiter"] vllm = ["vllm>=0.6.0"] drift-viewer = ["Pillow>=10", "PySide6>=6.6"] -dev = ["pytest", "black", "isort", "ruff", "mypy", "pre-commit"] +dev = ["pytest", "black", "isort", "ruff", "mypy", "pre-commit", "matplotlib"] [tool.setuptools.packages.find] where = ["."] diff --git a/tests/benchmarks/test_benchmark_deterministic_collectives.py b/tests/benchmarks/test_benchmark_deterministic_collectives.py new file mode 100644 index 000000000..8a39bad75 --- /dev/null +++ b/tests/benchmarks/test_benchmark_deterministic_collectives.py @@ -0,0 +1,263 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""CPU checks for collective benchmark timing boundaries.""" + +import json +from types import SimpleNamespace + +import pytest +import torch + +from benchmarks.distributed import benchmark_deterministic_collectives as benchmark + + +@pytest.mark.parametrize("warmup", [0, 2]) +@pytest.mark.parametrize("with_prepare", [False, True]) +def test_paired_samples_us_excludes_preparation(monkeypatch, warmup, with_prepare): + """Alternate each sample's order and reset NCCL inputs outside timing events.""" + clock_us = 0 + calls = [] + value = 1 + deterministic_durations = [17, 5, 9, 4, 12] + nccl_durations = [19, 8, 15, 6, 11] + deterministic_calls = 0 + nccl_calls = 0 + + class Event: + """Model CUDA events on a stream with a deterministic virtual clock.""" + + def __init__(self, *, enable_timing): + """Accept the same timing flag as a CUDA event.""" + assert enable_timing + self.timestamp = None + + def record(self): + """Record the stream's current virtual timestamp.""" + self.timestamp = clock_us + calls.append("event") + + def synchronize(self): + """Represent completion of the recorded event.""" + calls.append("end_sync") + + def elapsed_time(self, end): + """Return elapsed milliseconds, matching the CUDA event API.""" + return (end.timestamp - self.timestamp) / 1e3 + + def prepare(): + """Spend time resetting the mutable collective input.""" + nonlocal clock_us, value + clock_us += 1000 + value = 1 + calls.append("prepare") + + def deterministic(): + """Model an out-of-place reduction with varying sample durations.""" + nonlocal clock_us, deterministic_calls + clock_us += deterministic_durations[deterministic_calls] + deterministic_calls += 1 + calls.append("deterministic") + + def nccl(): + """Model an in-place reduction with varying sample durations.""" + nonlocal clock_us, value, nccl_calls + if with_prepare: + assert value == 1 + value *= 2 + clock_us += nccl_durations[nccl_calls] + nccl_calls += 1 + calls.append("nccl") + + monkeypatch.setattr(benchmark.torch.cuda, "Event", Event) + monkeypatch.setattr(benchmark.torch.cuda, "synchronize", lambda: calls.append("sync")) + measured = benchmark._paired_samples_us( + deterministic, nccl, warmup, 3, prepare_nccl=prepare if with_prepare else None + ) + + assert measured[0] == pytest.approx(deterministic_durations[warmup : warmup + 3]) + assert measured[1] == pytest.approx(nccl_durations[warmup : warmup + 3]) + expected = [] + for count, timed in ((warmup, False), (3, True)): + if timed: + expected.append("sync") + for sample in range(count): + order = ("deterministic", "nccl") if sample % 2 == 0 else ("nccl", "deterministic") + for name in order: + if name == "nccl" and with_prepare: + expected.append("prepare") + if timed: + expected.append("event") + expected.append(name) + if timed: + expected.extend(("event", "end_sync")) + assert calls == expected + + +def test_gather_samples_us_preserves_rank_order_and_precision(monkeypatch): + """Keep each rank's complete ordered samples without float32 rounding.""" + samples_by_rank = [[1.123456789, 3.987654321], [7.876543219, 2.234567891]] + + def gather(outputs, samples): + """Populate gathered tensors in global-rank order.""" + assert samples.dtype == torch.float64 + assert samples.tolist() == samples_by_rank[1] + for output, rank_samples in zip(outputs, samples_by_rank, strict=True): + output.copy_(torch.tensor(rank_samples, dtype=torch.float64)) + + cpu_torch = SimpleNamespace( + tensor=lambda values, *, device, dtype: torch.tensor(values, dtype=dtype), + float64=torch.float64, + empty_like=torch.empty_like, + ) + cpu_dist = SimpleNamespace(get_world_size=lambda: 2, all_gather=gather) + monkeypatch.setattr(benchmark, "torch", cpu_torch) + monkeypatch.setattr(benchmark, "dist", cpu_dist) + + assert benchmark._gather_samples_us(samples_by_rank[1]) == samples_by_rank + + +def test_reduction_callbacks_reuse_outputs_and_report_samples(monkeypatch, tmp_path): + """Reuse reduction buffers and report all ranks with their maximum medians.""" + outputs = {name: [] for name in ("reduce_scatter", "all_reduce")} + nccl_outputs = {name: [] for name in outputs} + inputs = [] + + class Collective: + """Model deterministic reductions on CPU with caller-provided outputs.""" + + def __init__(self, **kwargs): + """Accept the distributed benchmark configuration.""" + + def all_gather(self, *args, **kwargs): + """Reject gather calls in this reductions-only benchmark.""" + pytest.fail("only reduction callbacks should run") + + all_gather_many = all_gather + + def reduce_scatter(self, value, *, out, validate_signature): + """Write a two-rank reduction into the supplied output buffer.""" + assert validate_signature is False + outputs["reduce_scatter"].append(out) + return out.copy_(value.view(2, -1).sum(dim=0)) + + def all_reduce(self, value, *, out, validate_signature): + """Preserve the source while writing a simulated two-rank sum.""" + assert validate_signature is False + inputs.append(value.clone()) + assert out is not value + outputs["all_reduce"].append(out) + return out.copy_(value * 2) + + def close(self): + """Release the simulated collective.""" + + def nccl_scatter(out, value): + """Model NCCL's out-of-place reduce-scatter contract.""" + nccl_outputs["reduce_scatter"].append(out) + out.copy_(value.view(2, -1).sum(dim=0)) + + def nccl_reduce(value): + """Require a reset before each simulated in-place NCCL reduction.""" + torch.testing.assert_close(value, inputs[-1]) + nccl_outputs["all_reduce"].append(value) + value.mul_(2) + + measured = [] + + def measure(deterministic, nccl, warmup, iterations, *, prepare_nccl): + """Invoke the real benchmark callbacks repeatedly without GPU timing.""" + assert (warmup, iterations) == (1, 2) + for _ in range(warmup + iterations): + deterministic() + if prepare_nccl is not None: + prepare_nccl() + nccl() + # Offset the samples per call so each operation's row is distinguishable. + offset = 100.0 * len(measured) + measured.append(offset) + return [7.0 + offset, 19.0 + offset], [19.0 + offset, 17.0 + offset] + + def gather_samples(samples): + """Model a second rank whose samples differ from this rank's.""" + return [samples, [value + 1.0 for value in reversed(samples)]] + + cpu_torch = SimpleNamespace( + cuda=SimpleNamespace( + set_device=lambda rank: None, + get_device_properties=lambda rank: SimpleNamespace(name="test-gpu"), + ), + device=torch.device, + empty=lambda *shape, device=None, **kwargs: torch.empty(*shape, **kwargs), + randn=lambda *shape, device=None: torch.randn(*shape), + empty_like=torch.empty_like, + __version__="test-torch", + version=SimpleNamespace(cuda="test-cuda"), + ) + cpu_dist = SimpleNamespace( + init_process_group=lambda *args, **kwargs: None, + get_world_size=lambda: 2, + get_rank=lambda: 0, + all_gather_into_tensor=lambda *args: None, + reduce_scatter_tensor=nccl_scatter, + all_reduce=nccl_reduce, + destroy_process_group=lambda: None, + ) + monkeypatch.setenv("LOCAL_RANK", "0") + monkeypatch.setattr(benchmark, "torch", cpu_torch) + monkeypatch.setattr(benchmark, "dist", cpu_dist) + monkeypatch.setattr(benchmark, "DeterministicCollective", Collective) + monkeypatch.setattr(benchmark, "_paired_samples_us", measure) + monkeypatch.setattr(benchmark, "_gather_samples_us", gather_samples) + report_path = tmp_path / "nested" / "timings.json" + benchmark.main( + [ + "--size-bytes", + "32", + "--operations", + "reduce_scatter", + "all_reduce", + "--warmup", + "1", + "--iterations", + "2", + "--output", + str(report_path), + ] + ) + + for name in outputs: + assert len(outputs[name]) == len(nccl_outputs[name]) == 3 + assert all(out is outputs[name][0] for out in outputs[name]) + assert all(out is nccl_outputs[name][0] for out in nccl_outputs[name]) + assert outputs[name][0] is not nccl_outputs[name][0] + torch.testing.assert_close(outputs[name][0], nccl_outputs[name][0]) + assert all(torch.equal(value, inputs[0]) for value in inputs) + + report = json.loads(report_path.read_text()) + assert report["world_size"] == 2 + assert [row["operation"] for row in report["rows"]] == ["reduce_scatter", "all_reduce"] + assert (report["warmup"], report["iterations"]) == (1, 2) + assert [row["input_bytes_per_rank"] for row in report["rows"]] == [64, 32] + for row, offset in zip(report["rows"], measured, strict=True): + assert row["deterministic_samples_us"] == [ + [7.0 + offset, 19.0 + offset], + [20.0 + offset, 8.0 + offset], + ] + assert row["nccl_samples_us"] == [ + [19.0 + offset, 17.0 + offset], + [18.0 + offset, 20.0 + offset], + ] + assert row["deterministic_us"] == 14.0 + offset + assert row["nccl_us"] == 19.0 + offset + assert measured == [0.0, 100.0] + + +@pytest.mark.parametrize( + "argv", + [["--iterations", "0"], ["--warmup", "-1"], ["--size-bytes", "0"], ["--size-bytes", "33"]], +) +def test_parse_args_rejects_invalid_workloads(argv): + """Reject workloads that would produce no samples or truncated tensors.""" + with pytest.raises(SystemExit): + benchmark.parse_args(argv) diff --git a/tests/benchmarks/test_plot_deterministic_collectives.py b/tests/benchmarks/test_plot_deterministic_collectives.py new file mode 100644 index 000000000..ec3d9486d --- /dev/null +++ b/tests/benchmarks/test_plot_deterministic_collectives.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("matplotlib") + +from benchmarks.distributed import plot_deterministic_collectives as plotter # noqa: E402 + + +def _write_report( + path: Path, + world_size: int, + sizes: list[int], + latency: float, + slope: float = 1 / 1024, + nccl_offset: float = 0.0, +) -> None: + """Write a report whose latencies identify each size after sorting. + + ``slope`` sets how deterministic latency grows with size (negative: it shrinks, so the + latency order differs from the size order); ``nccl_offset`` tells two reports' NCCL apart. + """ + report = { + "world_size": world_size, + "gpu": "test GPU", + "torch": "test", + "cuda": "test", + "dtype": "float32", + "warmup": 1, + "iterations": 2, + "rows": [ + { + "operation": "all_gather", + "input_bytes_per_rank": size, + "deterministic_us": latency + slope * size, + "nccl_us": size / 1024 + nccl_offset, + } + for size in sizes + ], + } + path.write_text(json.dumps(report)) + + +@pytest.mark.parametrize( + ("after_world_size", "after_sizes", "after_metadata", "message"), + [ + (4, [1024, 2048], {}, "different world sizes"), + (2, [1024, 4096], {}, "different input sizes for all_gather"), + (2, [1024], {}, "different input sizes for all_gather"), + (2, [1024, 2048], {"gpu": "different GPU"}, "different gpu values"), + (2, [1024, 2048], {"dtype": "bf16"}, "different dtype values"), + ], +) +def test_mismatched_reports_fail_before_plotting( + tmp_path, monkeypatch, after_world_size, after_sizes, after_metadata, message +): + """Reject differing workloads or input sweeps before creating a figure.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + _write_report(before_path, 2, [1024, 2048], 10) + _write_report(after_path, after_world_size, after_sizes, 5) + report = json.loads(after_path.read_text()) + report.update(after_metadata) + after_path.write_text(json.dumps(report)) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if invalid reports reach the plotting stage.""" + pytest.fail("mismatched reports must be rejected before creating a figure") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises(SystemExit, match=message): + plotter.main() + + +@pytest.mark.parametrize("field", ["torch", "cuda", "dtype"]) +def test_environment_must_match_across_world_sizes(tmp_path, monkeypatch, field): + """Reject a multi-panel figure whose reports record different environments.""" + paths = {name: tmp_path / f"{name}.json" for name in ("b2", "a2", "b4", "a4")} + for name, world_size in (("b2", 2), ("a2", 2), ("b4", 4), ("a4", 4)): + _write_report(paths[name], world_size, [1024, 2048], 10 if name[0] == "b" else 5) + # Change both reports of the second pair: each pair stays consistent (so the pair check + # passes) and only the comparison with the first world size can reject it. + for name in ("b4", "a4"): + report = json.loads(paths[name].read_text()) + report[field] = "different" + paths[name].write_text(json.dumps(report)) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(paths["b2"]), + str(paths["b4"]), + "--after", + str(paths["a2"]), + str(paths["a4"]), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if reports from different environments reach the plotting stage.""" + pytest.fail("reports from different environments must be rejected") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises( + SystemExit, + match=f"{re.escape(str(paths['a4']))} and {re.escape(str(paths['a2']))} " + f"have different {field} values", + ): + plotter.main() + + +@pytest.mark.parametrize( + ("before_operation", "after_operation"), + [ + ("all_reduce", "all_gather"), + ("all_gather", "all_reduce"), + ("all_reduce", "all_reduce"), + ], +) +def test_missing_operation_fails_before_plotting( + tmp_path, monkeypatch, before_operation, after_operation +): + """Reject a selected operation absent from either or both reports.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + for path, operation in ((before_path, before_operation), (after_path, after_operation)): + _write_report(path, 2, [1024, 2048], 10) + report = json.loads(path.read_text()) + for row in report["rows"]: + row["operation"] = operation + path.write_text(json.dumps(report)) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(tmp_path / "figure.png"), + ], + ) + + def unexpected_subplots(*args, **kwargs): + """Fail if an empty selected series reaches the plotting stage.""" + pytest.fail("reports missing the selected operation must fail before plotting") + + monkeypatch.setattr(plotter.plt, "subplots", unexpected_subplots) + with pytest.raises(SystemExit, match="need rows for all_gather in both reports"): + plotter.main() + + +def test_reordered_rows_keep_latencies_aligned(tmp_path, monkeypatch): + """Plot matching sweeps in size order even when source rows are reordered.""" + before_path = tmp_path / "before.json" + after_path = tmp_path / "after.json" + output = tmp_path / "figure.png" + # Latency falls with size and the NCCL series differ, so sorting by latency, or taking the + # reference from the before report, would change the asserted series. + _write_report(before_path, 2, [2048, 1024], 100, slope=-1 / 1024, nccl_offset=100) + _write_report(after_path, 2, [1024, 2048], 50, slope=-1 / 1024) + monkeypatch.setattr( + sys, + "argv", + [ + "plot_deterministic_collectives.py", + "--before", + str(before_path), + "--after", + str(after_path), + "--output", + str(output), + ], + ) + figure, axes = plotter.plt.subplots(1, 1, squeeze=False) + monkeypatch.setattr(plotter.plt, "subplots", lambda *args, **kwargs: (figure, axes)) + try: + plotter.main() + before, after, reference = axes[0, 0].get_lines() + assert list(before.get_xdata()) == [1, 2] + assert list(before.get_ydata()) == [99, 98] + assert list(after.get_xdata()) == [1, 2] + assert list(after.get_ydata()) == [49, 48] + assert list(reference.get_ydata()) == [1, 2] + assert output.is_file() + finally: + plotter.plt.close(figure) diff --git a/tests/distributed/collectives/test_deterministic_all_gather_sizes.py b/tests/distributed/collectives/test_deterministic_all_gather_sizes.py new file mode 100644 index 000000000..8df644013 --- /dev/null +++ b/tests/distributed/collectives/test_deterministic_all_gather_sizes.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Deterministic all-gather is an exact rank-ordered copy at every size. + +The copy has a single-block path (gathered output <= 64 KiB, the C++ +``kAllGatherSingleBlockMaxBytes``) and a multi-block path, and moves 16-byte +vectors when pointers allow. This sweeps byte counts across the threshold, odd +sizes that leave a byte tail, and misaligned input and output pointers, and +compares every result with NCCL's all-gather. +""" + +from __future__ import annotations + +import os +import socket +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from rl_engine.distributed import create_deterministic_collective + +_WORLD_SIZE = max((n for n in (8, 4, 2) if n <= torch.cuda.device_count()), default=0) +_THRESHOLD = 64 * 1024 # kAllGatherSingleBlockMaxBytes + +pytestmark = [ + pytest.mark.cuda_only, + pytest.mark.skipif( + int(os.environ.get("WORLD_SIZE", "1")) != 1, + reason="this test owns its worker processes; run pytest directly", + ), + pytest.mark.skipif(_WORLD_SIZE < 2, reason="requires at least two visible CUDA GPUs"), +] + + +def _byte_sizes(world_size: int) -> list[int]: + """Return per-rank byte counts covering copy boundaries and vector tails.""" + per_rank_threshold = _THRESHOLD // world_size + return sorted( + { + 1, + 15, + 16, + 17, + 4095, + 32 * 1024 + 3, + 96 * 1024, + per_rank_threshold - 1, + per_rank_threshold, + per_rank_threshold + 1, + per_rank_threshold + 16, + 1024 * 1024, + 1024 * 1024 + 5, + } + ) + + +def _worker(rank: int, world_size: int, port: int) -> None: + """Compare rank-ordered copies with NCCL in an owned process group.""" + torch.cuda.set_device(rank) + device = torch.device("cuda", rank) + dist.init_process_group( + backend="nccl", + init_method=f"tcp://127.0.0.1:{port}", + rank=rank, + world_size=world_size, + timeout=timedelta(minutes=5), + ) + try: + with create_deterministic_collective( + device=device, max_size_bytes=2 * 1024 * 1024 + ) as collective: + generator = torch.Generator(device="cpu").manual_seed(1234 + rank) + for size in _byte_sizes(world_size): + for input_offset, output_offset in ((0, 0), (1, 0), (0, 3), (5, 7)): + source = torch.randint( + 0, 256, (size + input_offset,), generator=generator, dtype=torch.uint8 + ).to(device) + shard = source[input_offset:] + expected = torch.empty(size * world_size, dtype=torch.uint8, device=device) + dist.all_gather_into_tensor(expected, shard.contiguous()) + + tail_guard = 16 + storage = torch.full( + (size * world_size + output_offset + tail_guard,), + 0xA5, + dtype=torch.uint8, + device=device, + ) + out = storage[output_offset : output_offset + size * world_size] + returned = collective.all_gather(shard, out=out) + assert returned is out + assert torch.equal(out, expected), (size, input_offset, output_offset) + assert torch.all(storage[:output_offset] == 0xA5) + assert torch.all(storage[output_offset + size * world_size :] == 0xA5) + + if (input_offset, output_offset) == (0, 0) and size * world_size <= _THRESHOLD: + # The staged one-block kernel is a separate path from the fused entry. + staged_out = torch.full_like(expected, 0xA5) + collective._extension.deterministic_collective_stage( + collective._handle, shard + ) + collective._extension.deterministic_collective_all_gather( + collective._handle, staged_out + ) + assert torch.equal(staged_out, expected), (size, "staged") + dist.barrier() + finally: + dist.destroy_process_group() + + +def _find_free_port() -> int: + """Return an available loopback TCP port for the workers' rendezvous.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def test_all_gather_is_exact_across_sizes_and_alignments() -> None: + """Verify byte-exact output and prefix guards across sizes and offsets.""" + mp.spawn(_worker, args=(_WORLD_SIZE, _find_free_port()), nprocs=_WORLD_SIZE, join=True)