Skip to content
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
1c4985e
perf(cuda): vectorize the deterministic all-gather copy and right-siz…
fusheng-ji Oct 7, 2026
0707ac9
bench(cuda): deterministic all-gather before/after results on 2 and 8…
fusheng-ji Oct 7, 2026
6bfca95
perf(cuda): copy all peers' vectors in one flat loop when aligned
fusheng-ji Oct 7, 2026
7c79170
fix(bench): isolate NCCL timings and validate paired reports
fusheng-ji Oct 7, 2026
a290fac
fix(benchmarks): align collective timings and validate report workloads
fusheng-ji Oct 19, 2026
6128a72
fix(collectives): retain timing samples and right-size gather grid
fusheng-ji Oct 24, 2026
20e6354
fix(benchmarks): validate collective benchmark inputs and report meta…
fusheng-ji Oct 24, 2026
f7f0ff9
test(collectives): guard all-gather output prefix with a nonzero sent…
fusheng-ji Oct 24, 2026
af60420
docs: document the functions this change touches
fusheng-ji Oct 27, 2026
9a6192b
test(collectives): guard the bytes after the all-gather output
fusheng-ji Oct 28, 2026
2c97d5f
bench(collectives): report from the global rank and place devices by …
fusheng-ji Oct 28, 2026
746c62b
test(collectives): repair the plot fixture, reject a negative warmup …
fusheng-ji Oct 28, 2026
9333315
refactor(collectives): move the collective benchmarks and tests to th…
fusheng-ji Nov 2, 2026
0f5d0fa
Merge branch 'main' into fix/deterministic-collective-fast-copy
fusheng-ji Nov 2, 2026
bd9c1fb
fix(bench): skip signature validation in the timed deterministic coll…
fusheng-ji Nov 2, 2026
eb7d18a
docs(bench): state that --size-bytes is the reduce_scatter output per…
fusheng-ji Nov 2, 2026
c1a5269
docs(bench): explain why the process group is destroyed after the swe…
fusheng-ji Nov 3, 2026
a6b340c
Merge branch 'main' into fix/deterministic-collective-fast-copy
fusheng-ji Nov 3, 2026
28c9975
docs(bench): state that the all_gather_many row passes a single tensor
fusheng-ji Nov 3, 2026
49d4fb3
test(bench): declare matplotlib for dev, check plot environment acros…
fusheng-ji Nov 4, 2026
8c03d13
Merge branch 'main' into fix/deterministic-collective-fast-copy
Flink-ddd Nov 9, 2026
77bae7b
Merge branch 'main' into fix/deterministic-collective-fast-copy
fusheng-ji Nov 14, 2026
5882864
test(collectives): make the plot tests able to fail on the checks the…
fusheng-ji Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 213 additions & 0 deletions benchmarks/distributed/benchmark_deterministic_collectives.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
# SPDX-License-Identifier: Apache-2.0

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please move the collective benchmarks to benchmarks/distributed/, their unit tests to tests/benchmarks/, and the size/alignment test to tests/distributed/collectives/, updating imports and commands to follow #497; otherwise, LGTM.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in 9333315c. The benchmark and plot scripts are now in benchmarks/distributed/, their unit tests are in tests/benchmarks/, and test_deterministic_all_gather_sizes.py is in tests/distributed/collectives/. The imports, usage docstrings and the PR's Tests commands use the new paths. No code changed; the benchmark unit tests pass (10 passed, 1 skipped).

# Copyright (c) 2026 RL-Kernel Contributors

"""Benchmark the CUDA deterministic collectives across message sizes.

Example:

torchrun --standalone --nproc-per-node=8 \
benchmarks/distributed/benchmark_deterministic_collectives.py \
--output benchmarks/results/deterministic_collectives_b200.json

``--size-bytes`` is the per-rank input. The default sweep straddles the
single-block fast paths (64 KiB gathered output for ``all_gather``, 256 KiB
for the ``reduce_scatter`` input and ``all_reduce``). NCCL rows are
performance references only; they are not a correctness oracle.

JSON timing samples are grouped by global rank, in collection order. The
reported latency is the maximum of those ranks' median sample latencies.
"""

from __future__ import annotations

import argparse
import json
import os
import statistics
from functools import partial
from pathlib import Path
from typing import Callable, Sequence

import torch
import torch.distributed as dist

from rl_engine.distributed.collectives import DeterministicCollective

_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
_OPERATIONS = ("all_gather", "all_gather_many", "reduce_scatter", "all_reduce")
_DEFAULT_SIZES = [
1024,
8 * 1024,
16 * 1024,
32 * 1024,
64 * 1024,
96 * 1024,
128 * 1024,
192 * 1024,
256 * 1024,
512 * 1024,
1024 * 1024,
4 * 1024 * 1024,
]


def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
"""Parse the message-size sweep and distributed benchmark options."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--size-bytes", type=int, nargs="+", default=_DEFAULT_SIZES)
parser.add_argument("--dtype", choices=tuple(_DTYPES), default="bf16")
parser.add_argument("--operations", nargs="+", choices=_OPERATIONS, default=_OPERATIONS)
parser.add_argument("--warmup", type=int, default=20)
parser.add_argument("--iterations", type=int, default=100)
parser.add_argument("--output", type=Path)
args = parser.parse_args(argv)
if args.iterations <= 0:
parser.error("--iterations must be positive")
if args.warmup < 0:
parser.error("--warmup must not be negative")
element = torch.empty((), dtype=_DTYPES[args.dtype]).element_size()
if any(size <= 0 or size % element != 0 for size in args.size_bytes):
parser.error(
f"--size-bytes values must be positive and divisible by "
f"the {element}-byte dtype element size"
)
return args


def _paired_samples_us(
deterministic: Callable[[], object],
nccl: Callable[[], object],
warmup: int,
iterations: int,
*,
prepare_nccl: Callable[[], object] | None = None,
) -> tuple[list[float], list[float]]:
"""Collect alternating timing samples in microseconds, excluding resets."""
functions = (deterministic, nccl)
preparations = (None, prepare_nccl)
for sample in range(warmup):
order = (0, 1) if sample % 2 == 0 else (1, 0)
for index in order:
prepare = preparations[index]
if prepare is not None:
prepare()
functions[index]()
torch.cuda.synchronize()
samples: tuple[list[float], list[float]] = ([], [])
for sample in range(iterations):
order = (0, 1) if sample % 2 == 0 else (1, 0)
for index in order:
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
prepare = preparations[index]
if prepare is not None:
prepare()
start.record()
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
functions[index]()
end.record()
end.synchronize()
samples[index].append(start.elapsed_time(end) * 1e3)
return samples


def _gather_samples_us(samples: list[float]) -> list[list[float]]:
"""Gather timing samples in global-rank order without reducing their precision."""
tensor = torch.tensor(samples, device="cuda", dtype=torch.float64)
gathered = [torch.empty_like(tensor) for _ in range(dist.get_world_size())]
dist.all_gather(gathered, tensor)
return [rank_samples.tolist() for rank_samples in gathered]


def main(argv: Sequence[str] | None = None) -> None:
"""Measure selected collectives on every rank and save the rank-zero report."""
args = parse_args(argv)
local_rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local_rank)
dist.init_process_group("nccl", device_id=torch.device("cuda", local_rank))
rank = dist.get_rank()
world = dist.get_world_size()
dtype = _DTYPES[args.dtype]
element = torch.empty((), dtype=dtype).element_size()
capacity = max(args.size_bytes) * world
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
collective = DeterministicCollective(device=local_rank, max_size_bytes=capacity)
rows = []
try:
for size in args.size_bytes:
numel = size // element
x = torch.randn(numel, device="cuda").to(dtype)
gathered = torch.empty(numel * world, device="cuda", dtype=dtype)
reduce_in = torch.randn(numel * world, device="cuda").to(dtype)
deterministic_scattered = torch.empty_like(x)
deterministic_reduced = torch.empty_like(x)
nccl_scattered = torch.empty_like(x)
nccl_reduced = torch.empty_like(x)
cases: dict[str, tuple[Callable[[], object], Callable[[], object]]] = {
"all_gather": (
partial(collective.all_gather, x, out=gathered),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
partial(dist.all_gather_into_tensor, gathered, x),
),
"all_gather_many": (
partial(collective.all_gather_many, (x,)),
lambda out=gathered, src=x: dist.all_gather_into_tensor(
out.new_empty(out.shape), src
),
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
),
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
"reduce_scatter": (
partial(collective.reduce_scatter, reduce_in, out=deterministic_scattered),
partial(dist.reduce_scatter_tensor, nccl_scattered, reduce_in),
),
"all_reduce": (
partial(collective.all_reduce, x, out=deterministic_reduced),
partial(dist.all_reduce, nccl_reduced),
),
}
for op in args.operations:
deterministic, nccl = cases[op]
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
prepare = partial(nccl_reduced.copy_, x) if op == "all_reduce" else None
deterministic_samples, nccl_samples = _paired_samples_us(
deterministic, nccl, args.warmup, args.iterations, prepare_nccl=prepare
)
deterministic_samples_us = _gather_samples_us(deterministic_samples)
nccl_samples_us = _gather_samples_us(nccl_samples)
row = {
"operation": op,
"input_bytes_per_rank": (
reduce_in.numel() * element if op == "reduce_scatter" else numel * element
),
"deterministic_us": max(
statistics.median(samples) for samples in deterministic_samples_us
),
"nccl_us": max(statistics.median(samples) for samples in nccl_samples_us),
"deterministic_samples_us": deterministic_samples_us,
"nccl_samples_us": nccl_samples_us,
}
rows.append(row)
if rank == 0:
print(
f"{op:16s} {row['input_bytes_per_rank']:>9d} B/rank "
f"deterministic {row['deterministic_us']:8.1f} us "
f"nccl {row['nccl_us']:8.1f} us",
flush=True,
)
finally:
collective.close()
if rank == 0 and args.output is not None:
props = torch.cuda.get_device_properties(local_rank)
report = {
"kind": "deterministic_collectives_benchmark",
"world_size": world,
"dtype": args.dtype,
"gpu": props.name,
"warmup": args.warmup,
"iterations": args.iterations,
"torch": torch.__version__,
"cuda": torch.version.cuda,
"rows": rows,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n")
dist.destroy_process_group()


if __name__ == "__main__":
main()
130 changes: 130 additions & 0 deletions benchmarks/distributed/plot_deterministic_collectives.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2026 RL-Kernel Contributors

"""Plot before/after reports of ``benchmark_deterministic_collectives.py``.

python benchmarks/distributed/plot_deterministic_collectives.py \
--before before_w2.json before_w8.json --after after_w2.json after_w8.json \
--operation all_gather --output figure.png

One panel per world size: deterministic time before and after, with NCCL
(from the "after" run) as a reference. Needs only matplotlib.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402

BEFORE = "#eb6834"
AFTER = "#2a78d6"
REFERENCE = "#6f6e69"
INK = "#2b2b29"
GRID = "#e4e3dd"
SURFACE = "#fcfcfb"


def _rows(path: Path, operation: str) -> tuple[dict, list[dict]]:
"""Load a report and sort the selected operation's rows by input bytes per rank."""
report = json.loads(path.read_text())
rows = [row for row in report["rows"] if row["operation"] == operation]
return report, sorted(rows, key=lambda row: row["input_bytes_per_rank"])


def main() -> None:
"""Validate paired benchmark reports and save their latency comparison plot."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--before", type=Path, nargs="+", required=True)
parser.add_argument("--after", type=Path, nargs="+", required=True)
parser.add_argument("--operation", default="all_gather")
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if len(args.before) != len(args.after):
raise SystemExit("--before and --after need one report per world size each")

reports = []
for before_path, after_path in zip(args.before, args.after, strict=True):
before_report, before = _rows(before_path, args.operation)
after_report, after = _rows(after_path, args.operation)
if before_report["world_size"] != after_report["world_size"]:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
raise SystemExit(
f"{before_path} and {after_path} have different world sizes: "
f"{before_report['world_size']} != {after_report['world_size']}"
)
for field in ("gpu", "dtype", "torch", "cuda", "warmup", "iterations"):
if before_report[field] != after_report[field]:
raise SystemExit(
f"{before_path} and {after_path} have different {field} values: "
f"{before_report[field]!r} != {after_report[field]!r}"
)
before_sizes = [row["input_bytes_per_rank"] for row in before]
after_sizes = [row["input_bytes_per_rank"] for row in after]
if not before_sizes or not after_sizes:
raise SystemExit(
f"{before_path} and {after_path} need rows for {args.operation} in both reports"
)
if before_sizes != after_sizes:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
raise SystemExit(
f"{before_path} and {after_path} have different input sizes for "
f"{args.operation}: {before_sizes} != {after_sizes}"
)
reports.append((before, after_report, after))

fig, axes = plt.subplots(
1, len(args.before), figsize=(5.5 * len(args.before), 3.8), facecolor=SURFACE, squeeze=False
)
for ax, (before, after_report, after) in zip(axes[0], reports, strict=True):
sizes = [row["input_bytes_per_rank"] / 1024 for row in after]
Comment thread
fusheng-ji marked this conversation as resolved.
Resolved
series = (
("before", BEFORE, "-", [row["deterministic_us"] for row in before]),
("after", AFTER, "-", [row["deterministic_us"] for row in after]),
("NCCL (reference)", REFERENCE, "--", [row["nccl_us"] for row in after]),
)
for label, color, style, values in series:
ax.plot(
sizes,
values,
style,
color=color,
linewidth=2,
marker="o",
markersize=4,
label=label,
)
ax.set_xscale("log", base=2)
ax.set_yscale("log")
ax.set_facecolor(SURFACE)
ax.grid(color=GRID, linewidth=0.8)
ax.set_axisbelow(True)
for side in ("top", "right"):
ax.spines[side].set_visible(False)
ax.tick_params(colors=REFERENCE, labelsize=8)
ax.set_xlabel("input per rank (KiB)", color=REFERENCE, fontsize=9)
ax.set_ylabel("µs (slowest rank, lower is better)", color=REFERENCE, fontsize=9)
world = after_report["world_size"]
ax.set_title(
f"{args.operation}, {world} x {after_report['gpu']}", loc="left", fontsize=11, color=INK
)
ax.legend(frameon=False, fontsize=8, labelcolor=INK)
first = reports[0][1]
fig.text(
0.01,
0.01,
f"torch {first['torch']} · CUDA {first['cuda']} · {first['dtype']}",
fontsize=7,
color=REFERENCE,
)
fig.tight_layout(rect=(0, 0.04, 1, 1))
args.output.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(args.output, dpi=150, facecolor=SURFACE)
print(f"wrote {args.output}")


if __name__ == "__main__":
main()
Loading
Loading