Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 58 additions & 8 deletions fastvideo/distributed/parallel_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

import torch
import torch.distributed
import torch.distributed as dist
from torch.distributed import Backend, ProcessGroup, ReduceOp

import fastvideo.envs as envs
Expand Down Expand Up @@ -664,6 +665,26 @@ def init_world_group(ranks: list[int], local_rank: int, backend: str) -> GroupCo
)


def get_node_group() -> GroupCoordinator:
assert _NODE is not None, ("node group is not initialized")
return _NODE


def init_node_group(local_rank: int, backend: str):
cpu_group = get_world_group().cpu_group
node_ranks = get_same_node_ranks(cpu_group)
node_size = len(node_ranks)
# NOTE: assumes all nodes have the same number of GPUs.
# Heterogeneous clusters are not supported.
world_size = dist.get_world_size()
assert world_size % node_size == 0, (f"World size ({world_size}) must be divisible by "
f"node size ({node_size}) — heterogeneous clusters "
f"are not supported.")
all_node_ranks = [list(range(i * node_size, (i + 1) * node_size)) for i in range(world_size // node_size)]
global _NODE
_NODE = init_model_parallel_group(all_node_ranks, local_rank, backend)


def init_model_parallel_group(
group_ranks: list[list[int]],
local_rank: int,
Expand Down Expand Up @@ -743,6 +764,9 @@ def init_distributed_environment(
else:
assert _WORLD.world_size == torch.distributed.get_world_size(), (
"world group already initialized with a different world size")
# Init a group for each node
if _NODE is None:
init_node_group(local_rank, backend)


_SP: GroupCoordinator | None = None
Expand Down Expand Up @@ -959,6 +983,11 @@ def destroy_model_parallel() -> None:
_DP.destroy()
_DP = None

global _NODE
if _NODE:
_NODE.destroy()
_NODE = None


def destroy_distributed_environment() -> None:
global _WORLD
Expand All @@ -979,15 +1008,25 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
ray.shutdown()


def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]:
"""
This is a collective operation that returns if each rank is in the same node
as the source rank. It tests if processes are attached to the same
memory system (shared access to shared memory).
def get_same_node_ranks(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]:
"""Return the global ranks that share ``source_rank``'s node.

This is a collective operation that tests if processes are attached to the
same memory system (shared access to shared memory).

Replaces the boolean-returning ``is_the_same_node_as`` for callers that
want a rank list, not a per-rank boolean mask.

Args:
pg: the global process group to test
source_rank: the rank to test against

Returns:
A list of ranks that are in the same node as the source rank.
"""
if isinstance(pg, ProcessGroup):
assert torch.distributed.get_backend(pg) != torch.distributed.Backend.NCCL, (
"in_the_same_node_as should be tested with a non-NCCL group.")
"get_same_node_ranks should be tested with a non-NCCL group.")
# local rank inside the group
rank = torch.distributed.get_rank(group=pg)
world_size = torch.distributed.get_world_size(group=pg)
Expand Down Expand Up @@ -1032,7 +1071,7 @@ def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: i
if shm.buf[:len(magic_message)] == magic_message:
is_in_the_same_node[rank] = 1
except Exception as e:
logger.error("Error ignored in is_in_the_same_node: %s", e)
logger.error("Error ignored in get_same_node_ranks: %s", e)
finally:
if shm:
shm.close()
Expand All @@ -1056,7 +1095,18 @@ def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: i
rank_data = pg.broadcast_obj(is_in_the_same_node, src=i)
aggregated_data += rank_data

return [x == 1 for x in aggregated_data.tolist()]
return [ranks[i] for i, x in enumerate(aggregated_data.tolist()) if x == 1]


def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]:
"""Deprecated alias for ``get_same_node_ranks``.

NOTE: PR #572 changed the return value from ``list[bool]`` (a per-rank
boolean mask) to ``list[int]`` (global rank IDs that share the same node as
``source_rank``). Callers that treated the result as a boolean mask must
update.
"""
return get_same_node_ranks(pg, source_rank)


def initialize_tensor_parallel_group(tensor_model_parallel_size: int = 1,
Expand Down
53 changes: 53 additions & 0 deletions fastvideo/models/loader/benchmarks/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# Weight Loading Benchmarks

Benchmarks for measuring model weight loading speed from safetensors files.

## Scripts

### `benchmark_weight_loading.py`

Measures loading throughput for two modes:
- **`to_cpu=True, broadcast=False`**: load weights to CPU (memory-mapped, no broadcast)
- **`to_cpu=False, broadcast=True`**: load weights to GPU (rank 0 reads from disk, broadcasts to other ranks)

```bash
# Single GPU (no torchrun needed)
python fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \
--model-path /path/to/model --subfolder transformer

# Multi-GPU
torchrun --nproc_per_node=4 \
fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \
--model-path /path/to/model --subfolder transformer
```

### `benchmark_weight_loading_comparison.py`

A/B comparison of independent read vs rank-0 broadcast, with both sync and async broadcast variants. Use this to measure the speedup from distributed weight loading.

```bash
torchrun --nproc_per_node=4 \
fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py \
--model-path /path/to/model --subfolder transformer
```

## Options

| Flag | Default | Description |
|------|---------|-------------|
| `--model-path` | required | Local directory containing `.safetensors` files |
| `--subfolder` | none | Subfolder within model-path (e.g. `transformer`, `text_encoder`) |
| `--warmup` | 1 | Warmup iterations before timing |
| `--repeats` | 3 | Timed iterations |

## Example results

HunyuanVideo transformer (25.64 GB) on NVIDIA B200:

| GPUs | Independent read | Sync broadcast | Speedup |
|------|-----------------|----------------|---------|
| 1 | 4.12s (6.22 GB/s) | 4.12s (6.22 GB/s) | 1.00x |
| 2 | 4.32s (5.94 GB/s) | 4.32s (5.94 GB/s) | 1.00x |
| 4 | 5.62s (4.56 GB/s) | 4.54s (5.65 GB/s) | 1.24x |

The broadcast approach eliminates disk I/O contention as GPU count increases.
1 change: 1 addition & 0 deletions fastvideo/models/loader/benchmarks/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# SPDX-License-Identifier: Apache-2.0
156 changes: 156 additions & 0 deletions fastvideo/models/loader/benchmarks/benchmark_weight_loading.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
"""Benchmark for model weight loading speed.

Measures the time to load model weights from safetensors files using
different strategies (CPU vs GPU, broadcast vs independent).

Usage (single GPU):
python fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \
--model-path /path/to/model

Usage (multi-GPU, e.g. 4 GPUs):
torchrun --nproc_per_node=4 \
fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \
--model-path /path/to/model
"""

import argparse
import os
import time

import torch
import torch.distributed as dist

from fastvideo.distributed.parallel_state import (
cleanup_dist_env_and_memory,
get_node_group,
maybe_init_distributed_environment_and_model_parallel,
)
from fastvideo.logger import init_logger
from fastvideo.models.loader.weight_utils import (
resolve_safetensors_files,
safetensors_weights_iterator,
)

logger = init_logger(__name__)


def benchmark_loading(
files: list[str],
to_cpu: bool,
broadcast: bool,
warmup: int,
repeats: int,
label: str,
) -> None:
"""Run the weight loading benchmark and print results."""
rank = dist.get_rank() if dist.is_initialized() else 0
node_group = get_node_group()

# Count total params and bytes on first pass
total_params = 0
total_bytes = 0
for name, tensor in safetensors_weights_iterator(
files, to_cpu=to_cpu, broadcast=broadcast):
total_params += 1
total_bytes += tensor.nelement() * tensor.element_size()

if rank == 0:
logger.info("[%s] %d tensors, %.2f GB total",
label, total_params, total_bytes / 1e9)

# Warmup
for _ in range(warmup):
for _ in safetensors_weights_iterator(
files, to_cpu=to_cpu, broadcast=broadcast):
pass
if dist.is_initialized():
dist.barrier()

# Timed runs
times = []
for i in range(repeats):
if dist.is_initialized():
dist.barrier()
torch.cuda.synchronize() if torch.cuda.is_available() else None

t0 = time.perf_counter()
for _ in safetensors_weights_iterator(
files, to_cpu=to_cpu, broadcast=broadcast):
pass
torch.cuda.synchronize() if torch.cuda.is_available() else None

if dist.is_initialized():
dist.barrier()
elapsed = time.perf_counter() - t0
times.append(elapsed)

if rank == 0:
avg = sum(times) / len(times)
best = min(times)
throughput = total_bytes / avg / 1e9
logger.info(
"[%s] avg %.3fs | best %.3fs | throughput %.2f GB/s "
"(over %d runs, %d warmup, %d GPU(s), node_size=%d)",
label, avg, best, throughput,
repeats, warmup, node_group.world_size, node_group.world_size,
)


def main() -> None:
parser = argparse.ArgumentParser(
description="Benchmark model weight loading speed")
parser.add_argument(
"--model-path", type=str, required=True,
help="Path to a local model directory containing .safetensors files")
parser.add_argument(
"--subfolder", type=str, default=None,
help="Subfolder within model-path to load (e.g. 'transformer', "
"'text_encoder'). If not set, loads from model-path directly.")
parser.add_argument(
"--warmup", type=int, default=1,
help="Number of warmup iterations (default: 1)")
parser.add_argument(
"--repeats", type=int, default=3,
help="Number of timed iterations (default: 3)")
args = parser.parse_args()

load_path = args.model_path
if args.subfolder:
load_path = os.path.join(load_path, args.subfolder)

files = resolve_safetensors_files(load_path)

# Set default env vars for single-GPU standalone mode (no torchrun)
if "MASTER_ADDR" not in os.environ:
os.environ["MASTER_ADDR"] = "localhost"
if "MASTER_PORT" not in os.environ:
os.environ["MASTER_PORT"] = "29500"
if "RANK" not in os.environ:
os.environ["RANK"] = "0"
if "WORLD_SIZE" not in os.environ:
os.environ["WORLD_SIZE"] = "1"

maybe_init_distributed_environment_and_model_parallel(1, 1)
rank = dist.get_rank() if dist.is_initialized() else 0
world_size = dist.get_world_size() if dist.is_initialized() else 1

if rank == 0:
logger.info("Model path: %s", load_path)
logger.info("Safetensors files: %d", len(files))
logger.info("World size: %d", world_size)

# Benchmark: load to CPU (no broadcast, each rank reads independently)
benchmark_loading(files, to_cpu=True, broadcast=False, warmup=args.warmup,
repeats=args.repeats, label="to_cpu=True")

# Benchmark: load to GPU (rank 0 reads, broadcasts to others)
if torch.cuda.is_available():
benchmark_loading(files, to_cpu=False, broadcast=True, warmup=args.warmup,
repeats=args.repeats, label="to_cpu=False (broadcast)")

cleanup_dist_env_and_memory()


if __name__ == "__main__":
main()
Loading
Loading