From fe0c86704261f0532d00db75f3bab867bcd05082 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Tue, 14 Apr 2026 15:24:41 -0700 Subject: [PATCH 01/11] Revert distributed load: re-apply distributed weight loading feature Squashed from: revert-571-will/revert_distributed_load Original commits: fix dist load device, fix fsdp to cpu, fix group, fix typos, remove ckpt Co-Authored-By: Edenzzzz --- .../Data-free/distill_dmd_t2v_5B_lora.sh | 146 ++++++++++++++++++ fastvideo/distributed/parallel_state.py | 29 +++- fastvideo/layers/layernorm.py | 8 +- fastvideo/models/loader/fsdp_load.py | 1 - fastvideo/models/loader/weight_utils.py | 61 ++++++-- fastvideo/pipelines/training/__init__.py | 6 + 6 files changed, 237 insertions(+), 14 deletions(-) create mode 100644 examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh diff --git a/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh b/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh new file mode 100644 index 0000000000..8a215b888c --- /dev/null +++ b/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh @@ -0,0 +1,146 @@ +#!/bin/bash +#SBATCH --job-name=t2v +#SBATCH --partition=main +#SBATCH --nodes=8 +#SBATCH --ntasks=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=dmd_Wan2.2/t2v_g2e5_f1e5_%j.out +#SBATCH --error=dmd_Wan2.2/t2v_g2e5_f1e5_%j.err +#SBATCH --exclusive +set -e -x + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate your_env + +# Basic Info +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +# different cache dir for different processes +export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID +export TOKENIZERS_PARALLELISM=false +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN +export WANDB_API_KEY=your_wandb_api_key +# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +# Configs +NUM_GPUS=8 +MODEL_PATH="Wan-AI/Wan2.2-TI2V-5B-Diffusers" +DATA_DIR=your_data_dir +VALIDATION_DIR=your_validation_path #(example:validation_64.json) +# export CUDA_VISIBLE_DEVICES=4,5 +# IP=[MASTER NODE IP] + +# Training arguments +training_args=( + --tracker_project_name Wan_distillation + --output_dir "your_output_dir" + --max_train_steps 4000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 31 + --num_height 704 + --num_width 1280 + --num_frames 121 + --enable_gradient_checkpointing_type "full" + --lora_rank 32 + --lora_training True +) + +# Parallel arguments +parallel_args=( + --num_gpus 64 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 64 + --hsdp_shard_dim 1 +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DIR" + --validation_steps 200 + --validation_sampling_steps "3" + --validation_guidance_scale "6.0" # not used for dmd inference +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-4 + --lr_scheduler "cosine_with_min_lr" + --min_lr_ratio 0.5 + --lr_warmup_steps 100 + --fake_score_learning_rate 1e-5 + --fake_score_lr_scheduler "cosine_with_min_lr" + --mixed_precision "bf16" + --training_state_checkpointing_steps 500 + --weight_only_checkpointing_steps 200 + --weight_decay 0.01 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --ema_start_step 0 + --flow_shift 5 + --seed 1000 +) + +# DMD arguments +dmd_args=( + --dmd_denoising_steps '1000,757,522' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --generator_update_interval 5 + --real_score_guidance_scale 3 + --simulate_generator_forward + --log_visualization # disable if oom +) + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node $NUM_GPUS \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/wan_distillation_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" \ No newline at end of file diff --git a/fastvideo/distributed/parallel_state.py b/fastvideo/distributed/parallel_state.py index 6f28cde786..d6d32535c4 100644 --- a/fastvideo/distributed/parallel_state.py +++ b/fastvideo/distributed/parallel_state.py @@ -36,6 +36,7 @@ import torch import torch.distributed +import torch.distributed as dist from torch.distributed import Backend, ProcessGroup, ReduceOp import fastvideo.envs as envs @@ -664,6 +665,23 @@ def init_world_group(ranks: list[int], local_rank: int, backend: str) -> GroupCo ) +def get_node_group() -> GroupCoordinator: + assert _NODE is not None, ("node group is not initialized") + return _NODE + + +def init_node_group(local_rank: int, backend: str): + cpu_group = get_world_group().cpu_group + node_ranks = is_the_same_node_as(cpu_group) + node_size = len(node_ranks) + all_node_ranks = [ + list(range(i * node_size, (i + 1) * node_size)) + for i in range(dist.get_world_size() // node_size) + ] + global _NODE + _NODE = init_model_parallel_group(all_node_ranks, local_rank, backend) + + def init_model_parallel_group( group_ranks: list[list[int]], local_rank: int, @@ -743,6 +761,8 @@ def init_distributed_environment( else: assert _WORLD.world_size == torch.distributed.get_world_size(), ( "world group already initialized with a different world size") + # Init a group for each node + init_node_group(local_rank, backend) _SP: GroupCoordinator | None = None @@ -981,9 +1001,14 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False): def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]: """ - This is a collective operation that returns if each rank is in the same node + This is a collective operation that returns ranks that are in the same node as the source rank. It tests if processes are attached to the same memory system (shared access to shared memory). + Args: + pg: the global process group to test + source_rank: the rank to test against + Returns: + A list of ranks that are in the same node as the source rank. """ if isinstance(pg, ProcessGroup): assert torch.distributed.get_backend(pg) != torch.distributed.Backend.NCCL, ( @@ -1056,7 +1081,7 @@ def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: i rank_data = pg.broadcast_obj(is_in_the_same_node, src=i) aggregated_data += rank_data - return [x == 1 for x in aggregated_data.tolist()] + return [i for i, x in enumerate(aggregated_data.tolist()) if x == 1] def initialize_tensor_parallel_group(tensor_model_parallel_size: int = 1, diff --git a/fastvideo/layers/layernorm.py b/fastvideo/layers/layernorm.py index b74059888c..d019d3c29a 100644 --- a/fastvideo/layers/layernorm.py +++ b/fastvideo/layers/layernorm.py @@ -5,6 +5,7 @@ import torch import torch.nn as nn import torch.nn.functional as F +from torch.distributed.tensor import DTensor from fastvideo.layers.custom_op import CustomOp @@ -76,7 +77,12 @@ def forward_native( x = x * torch.rsqrt(variance + self.variance_epsilon) x = x.to(orig_dtype) if self.has_weight: - x = x * self.weight + # TODO(wenxuan): When using CPU offload, FSDP has a bug that doesn't unwrap DTensor in final_layer_norm. + # Report this + if isinstance(self.weight, DTensor): + x = x * self.weight.to_local().to(x.device) + else: + x = x * self.weight if residual is None: return x else: diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index d4e83d127c..62a00fdb21 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -210,7 +210,6 @@ def maybe_load_fsdp_model( def shard_model( model, - *, cpu_offload: bool, reshard_after_forward: bool = True, mp_policy: MixedPrecisionPolicy | None = MixedPrecisionPolicy(), # noqa diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index 3e6fa91ba8..225333493b 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -11,9 +11,11 @@ import filelock import huggingface_hub.constants import torch +import torch.distributed as dist from safetensors.torch import safe_open from tqdm.auto import tqdm +from fastvideo.distributed.parallel_state import get_node_group from fastvideo.distributed import get_local_torch_device from fastvideo.logger import init_logger @@ -119,12 +121,24 @@ def filter_files_not_needed_for_inference( def safetensors_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = True, + to_cpu: bool = False, + async_broadcast: bool = False ) -> Generator[tuple[str, torch.Tensor], None, None]: - """Iterate over the weights in the model safetensor files.""" - enable_tqdm = not torch.distributed.is_initialized( - ) or torch.distributed.get_rank() == 0 - device = "cpu" if to_cpu else str(get_local_torch_device()) + """Iterate over the weights in the model safetensor files. + Args: + hf_weights_files: List of safetensor files to load. + to_cpu: Whether to load the weights to CPU. If False, will load to the GPU device bound to the current process. + async_broadcast: Whether to overlap loading from disk and broadcasting to other ranks. If True, + must iterate over all the weights before use. Only use if to_cpu is False. + """ + node_group = get_node_group() + local_rank = node_group.local_rank + device = f"cuda:{local_rank}" if not to_cpu else "cpu" + enable_tqdm = not torch.distributed.is_initialized() or local_rank == 0 + assert not (async_broadcast + and to_cpu), "Cannot broadcast weights when loading to CPU" + + handles = [] for st_file in tqdm( hf_weights_files, desc="Loading safetensors checkpoint shards", @@ -133,18 +147,45 @@ def safetensors_weights_iterator( ): with safe_open(st_file, framework="pt", device=device) as f: for name in f.keys(): # noqa: SIM118 - param = f.get_tensor(name) + if to_cpu: + param = f.get_tensor(name) + else: + if local_rank == 0: + param = f.get_tensor(name) + else: + shape = f.get_slice(name).get_shape() + param = torch.empty(shape, device=device) + # broadcast to local ranks + # TODO(Wenxuan): scatter instead of broadcast + if node_group.world_size > 1: + group = node_group.device_group + if async_broadcast: + handle = dist.broadcast(param, + src=dist.get_global_rank( + group, 0), + async_op=True, + group=group) + handles.append(handle) + else: + dist.broadcast(param, + src=dist.get_global_rank(group, 0), + group=group) yield name, param + if async_broadcast: + for handle in handles: + handle.wait() + def pt_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = True, + to_cpu: bool = True # default to CPU for text encoder ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model bin/pt files.""" - device = "cpu" if to_cpu else str(get_local_torch_device()) - enable_tqdm = not torch.distributed.is_initialized( - ) or torch.distributed.get_rank() == 0 + node_group = get_node_group() + local_rank = node_group.local_rank + device = f"cuda:{local_rank}" if not to_cpu else "cpu" + enable_tqdm = not torch.distributed.is_initialized() or local_rank == 0 for bin_file in tqdm( hf_weights_files, desc="Loading pt checkpoint shards", diff --git a/fastvideo/pipelines/training/__init__.py b/fastvideo/pipelines/training/__init__.py index e69de29bb2..30043cf114 100644 --- a/fastvideo/pipelines/training/__init__.py +++ b/fastvideo/pipelines/training/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Training pipelines for fastvideo. + +This package contains pipelines for training diffusion models. +""" From 5ced53137ba6f2dd54595d8f8e9a06ac03b73d2d Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Tue, 14 Apr 2026 16:08:40 -0700 Subject: [PATCH 02/11] fix distributed weight loading bugs and add loading benchmark - Fix dtype mismatch: non-rank-0 torch.empty now uses correct dtype from safetensors metadata instead of defaulting to float32 - Fix async broadcast handle accumulation across safetensors files - Add _NODE cleanup to destroy_model_parallel() to prevent stale state - Guard init_node_group() to skip redundant collective on re-entry - Preserve to_cpu=True for FSDP training path in fsdp_load.py - Add weight loading benchmark scripts for measuring broadcast speedup Co-Authored-By: Edenzzzz --- fastvideo/distributed/parallel_state.py | 11 +- .../models/loader/benchmarks/__init__.py | 1 + .../benchmarks/benchmark_weight_loading.py | 168 +++++++++++++ .../benchmark_weight_loading_comparison.py | 221 ++++++++++++++++++ fastvideo/models/loader/fsdp_load.py | 2 +- fastvideo/models/loader/weight_utils.py | 22 +- 6 files changed, 419 insertions(+), 6 deletions(-) create mode 100644 fastvideo/models/loader/benchmarks/__init__.py create mode 100644 fastvideo/models/loader/benchmarks/benchmark_weight_loading.py create mode 100644 fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py diff --git a/fastvideo/distributed/parallel_state.py b/fastvideo/distributed/parallel_state.py index d6d32535c4..7fd95051e8 100644 --- a/fastvideo/distributed/parallel_state.py +++ b/fastvideo/distributed/parallel_state.py @@ -675,8 +675,7 @@ def init_node_group(local_rank: int, backend: str): node_ranks = is_the_same_node_as(cpu_group) node_size = len(node_ranks) all_node_ranks = [ - list(range(i * node_size, (i + 1) * node_size)) - for i in range(dist.get_world_size() // node_size) + list(range(i * node_size, (i + 1) * node_size)) for i in range(dist.get_world_size() // node_size) ] global _NODE _NODE = init_model_parallel_group(all_node_ranks, local_rank, backend) @@ -762,7 +761,8 @@ def init_distributed_environment( assert _WORLD.world_size == torch.distributed.get_world_size(), ( "world group already initialized with a different world size") # Init a group for each node - init_node_group(local_rank, backend) + if _NODE is None: + init_node_group(local_rank, backend) _SP: GroupCoordinator | None = None @@ -979,6 +979,11 @@ def destroy_model_parallel() -> None: _DP.destroy() _DP = None + global _NODE + if _NODE: + _NODE.destroy() + _NODE = None + def destroy_distributed_environment() -> None: global _WORLD diff --git a/fastvideo/models/loader/benchmarks/__init__.py b/fastvideo/models/loader/benchmarks/__init__.py new file mode 100644 index 0000000000..9881313609 --- /dev/null +++ b/fastvideo/models/loader/benchmarks/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py new file mode 100644 index 0000000000..229d514192 --- /dev/null +++ b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Benchmark for model weight loading speed. + +Measures the time to load model weights from safetensors files using +different strategies (CPU vs GPU, broadcast vs independent). + +Usage (single GPU): + python fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \ + --model-path /path/to/model + +Usage (multi-GPU, e.g. 4 GPUs): + torchrun --nproc_per_node=4 \ + fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \ + --model-path /path/to/model +""" + +import argparse +import glob +import os +import time + +import torch +import torch.distributed as dist + +from fastvideo.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + get_node_group, + maybe_init_distributed_environment_and_model_parallel, +) +from fastvideo.logger import init_logger +from fastvideo.models.loader.weight_utils import ( + filter_duplicate_safetensors_files, + safetensors_weights_iterator, +) + +SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json" + +logger = init_logger(__name__) + + +def resolve_safetensors_files(model_path: str) -> list[str]: + """Discover safetensors files in a model directory.""" + files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) + if not files: + raise FileNotFoundError( + f"No .safetensors files found in {model_path}") + index_file = os.path.join(model_path, SAFE_WEIGHTS_INDEX_NAME) + if os.path.exists(index_file): + files = filter_duplicate_safetensors_files( + files, model_path, SAFE_WEIGHTS_INDEX_NAME) + return files + + +def benchmark_loading( + files: list[str], + to_cpu: bool, + warmup: int, + repeats: int, + label: str, +) -> None: + """Run the weight loading benchmark and print results.""" + rank = dist.get_rank() if dist.is_initialized() else 0 + node_group = get_node_group() + + # Count total params and bytes on first pass + total_params = 0 + total_bytes = 0 + for name, tensor in safetensors_weights_iterator(files, to_cpu=to_cpu): + total_params += 1 + total_bytes += tensor.nelement() * tensor.element_size() + + if rank == 0: + logger.info("[%s] %d tensors, %.2f GB total", + label, total_params, total_bytes / 1e9) + + # Warmup + for _ in range(warmup): + for _ in safetensors_weights_iterator(files, to_cpu=to_cpu): + pass + if dist.is_initialized(): + dist.barrier() + + # Timed runs + times = [] + for i in range(repeats): + if dist.is_initialized(): + dist.barrier() + torch.cuda.synchronize() if torch.cuda.is_available() else None + + t0 = time.perf_counter() + for _ in safetensors_weights_iterator(files, to_cpu=to_cpu): + pass + torch.cuda.synchronize() if torch.cuda.is_available() else None + + if dist.is_initialized(): + dist.barrier() + elapsed = time.perf_counter() - t0 + times.append(elapsed) + + if rank == 0: + avg = sum(times) / len(times) + best = min(times) + throughput = total_bytes / avg / 1e9 + logger.info( + "[%s] avg %.3fs | best %.3fs | throughput %.2f GB/s " + "(over %d runs, %d warmup, %d GPU(s), node_size=%d)", + label, avg, best, throughput, + repeats, warmup, node_group.world_size, node_group.world_size, + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Benchmark model weight loading speed") + parser.add_argument( + "--model-path", type=str, required=True, + help="Path to a local model directory containing .safetensors files") + parser.add_argument( + "--subfolder", type=str, default=None, + help="Subfolder within model-path to load (e.g. 'transformer', " + "'text_encoder'). If not set, loads from model-path directly.") + parser.add_argument( + "--warmup", type=int, default=1, + help="Number of warmup iterations (default: 1)") + parser.add_argument( + "--repeats", type=int, default=3, + help="Number of timed iterations (default: 3)") + args = parser.parse_args() + + load_path = args.model_path + if args.subfolder: + load_path = os.path.join(load_path, args.subfolder) + + files = resolve_safetensors_files(load_path) + + # Set default env vars for single-GPU standalone mode (no torchrun) + if "MASTER_ADDR" not in os.environ: + os.environ["MASTER_ADDR"] = "localhost" + if "MASTER_PORT" not in os.environ: + os.environ["MASTER_PORT"] = "29500" + if "RANK" not in os.environ: + os.environ["RANK"] = "0" + if "WORLD_SIZE" not in os.environ: + os.environ["WORLD_SIZE"] = "1" + + maybe_init_distributed_environment_and_model_parallel(1, 1) + rank = dist.get_rank() if dist.is_initialized() else 0 + world_size = dist.get_world_size() if dist.is_initialized() else 1 + + if rank == 0: + logger.info("Model path: %s", load_path) + logger.info("Safetensors files: %d", len(files)) + logger.info("World size: %d", world_size) + + # Benchmark: load to CPU (no broadcast, each rank reads independently) + benchmark_loading(files, to_cpu=True, warmup=args.warmup, + repeats=args.repeats, label="to_cpu=True") + + # Benchmark: load to GPU (rank 0 reads, broadcasts to others) + if torch.cuda.is_available(): + benchmark_loading(files, to_cpu=False, warmup=args.warmup, + repeats=args.repeats, label="to_cpu=False (broadcast)") + + cleanup_dist_env_and_memory() + + +if __name__ == "__main__": + main() diff --git a/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py b/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py new file mode 100644 index 0000000000..c6ccb355b7 --- /dev/null +++ b/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A/B benchmark: independent-read vs rank-0-broadcast weight loading. + +Compares two strategies: + - "before" (independent): every rank reads safetensors from disk to GPU + - "after" (broadcast): rank 0 reads from disk, broadcasts to other ranks + +Usage: + # 1 GPU + torchrun --nproc_per_node=1 fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py \ + --model-path /path/to/model --subfolder transformer + + # 2 GPUs + torchrun --nproc_per_node=2 fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py \ + --model-path /path/to/model --subfolder transformer + + # 4 GPUs + torchrun --nproc_per_node=4 fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py \ + --model-path /path/to/model --subfolder transformer +""" + +import argparse +import glob +import os +import time + +import torch +import torch.distributed as dist +from safetensors.torch import safe_open +from tqdm.auto import tqdm + +from fastvideo.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + get_node_group, + maybe_init_distributed_environment_and_model_parallel, +) +from fastvideo.models.loader.weight_utils import ( + SAFETENSORS_TO_TORCH_DTYPE, + filter_duplicate_safetensors_files, +) + +SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json" + + +def resolve_safetensors_files(model_path: str) -> list[str]: + files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) + if not files: + raise FileNotFoundError( + f"No .safetensors files found in {model_path}") + index_file = os.path.join(model_path, SAFE_WEIGHTS_INDEX_NAME) + if os.path.exists(index_file): + files = filter_duplicate_safetensors_files( + files, model_path, SAFE_WEIGHTS_INDEX_NAME) + return files + + +def load_independent(files: list[str], device: str): + """Before-PR behavior: every rank reads every tensor from disk to GPU.""" + for st_file in files: + with safe_open(st_file, framework="pt", device=device) as f: + for name in f.keys(): + param = f.get_tensor(name) + yield name, param + + +def load_broadcast(files: list[str], device: str, node_group, + async_op: bool = False): + """After-PR behavior: rank 0 reads from disk, broadcasts to other ranks.""" + local_rank = node_group.local_rank + handles = [] + for st_file in files: + with safe_open(st_file, framework="pt", device=device) as f: + for name in f.keys(): + if local_rank == 0: + param = f.get_tensor(name) + else: + sl = f.get_slice(name) + shape = sl.get_shape() + dtype = SAFETENSORS_TO_TORCH_DTYPE[sl.get_dtype()] + param = torch.empty(shape, device=device, dtype=dtype) + if node_group.world_size > 1: + group = node_group.device_group + if async_op: + handle = dist.broadcast( + param, + src=dist.get_global_rank(group, 0), + async_op=True, + group=group, + ) + handles.append(handle) + else: + dist.broadcast( + param, + src=dist.get_global_rank(group, 0), + group=group, + ) + yield name, param + if async_op: + for handle in handles: + handle.wait() + handles.clear() + + +def run_benchmark(label, loader_fn, files, warmup, repeats): + rank = dist.get_rank() + + # Count tensors/bytes on first pass + total_params = 0 + total_bytes = 0 + for name, tensor in loader_fn(files): + total_params += 1 + total_bytes += tensor.nelement() * tensor.element_size() + dist.barrier() + + # Warmup + for _ in range(warmup): + for _ in loader_fn(files): + pass + dist.barrier() + + # Timed runs + times = [] + for _ in range(repeats): + dist.barrier() + torch.cuda.synchronize() + t0 = time.perf_counter() + for _ in loader_fn(files): + pass + torch.cuda.synchronize() + dist.barrier() + times.append(time.perf_counter() - t0) + + if rank == 0: + avg = sum(times) / len(times) + best = min(times) + throughput = total_bytes / avg / 1e9 + print( + f" {label:30s} " + f"avg {avg:.3f}s best {best:.3f}s " + f"throughput {throughput:.2f} GB/s " + f"({total_params} tensors, {total_bytes/1e9:.2f} GB)" + ) + return avg, best, throughput, total_bytes + return None, None, None, None + + +def main(): + parser = argparse.ArgumentParser( + description="A/B benchmark: independent vs broadcast weight loading") + parser.add_argument("--model-path", type=str, required=True) + parser.add_argument("--subfolder", type=str, default=None) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + + load_path = args.model_path + if args.subfolder: + load_path = os.path.join(load_path, args.subfolder) + + files = resolve_safetensors_files(load_path) + + maybe_init_distributed_environment_and_model_parallel(1, 1) + rank = dist.get_rank() + world_size = dist.get_world_size() + node_group = get_node_group() + local_rank = node_group.local_rank + device = f"cuda:{local_rank}" + + if rank == 0: + print(f"\n{'='*72}") + print(f"Weight Loading Benchmark: {load_path}") + print(f"GPUs: {world_size} | safetensors files: {len(files)} " + f"| warmup: {args.warmup} | repeats: {args.repeats}") + print(f"{'='*72}") + + # --- Before PR: every rank reads independently --- + before_avg, before_best, before_tp, total_bytes = run_benchmark( + "before (independent read)", + lambda f: load_independent(f, device), + files, args.warmup, args.repeats, + ) + + # Clear GPU memory between benchmarks + torch.cuda.empty_cache() + dist.barrier() + + # --- After PR: rank 0 reads, sync broadcasts --- + after_sync_avg, after_sync_best, after_sync_tp, _ = run_benchmark( + "after (sync broadcast)", + lambda f: load_broadcast(f, device, node_group, async_op=False), + files, args.warmup, args.repeats, + ) + + # Clear GPU memory between benchmarks + torch.cuda.empty_cache() + dist.barrier() + + # --- After PR: rank 0 reads, async broadcasts --- + after_async_avg, after_async_best, after_async_tp, _ = run_benchmark( + "after (async broadcast)", + lambda f: load_broadcast(f, device, node_group, async_op=True), + files, args.warmup, args.repeats, + ) + + if rank == 0: + print(f"{'─'*72}") + if before_avg and after_sync_avg: + sync_speedup = before_avg / after_sync_avg + print(f" sync speedup: {sync_speedup:.2f}x " + f"(saved {before_avg - after_sync_avg:.3f}s)") + if before_avg and after_async_avg: + async_speedup = before_avg / after_async_avg + print(f" async speedup: {async_speedup:.2f}x " + f"(saved {before_avg - after_async_avg:.3f}s)") + print(f"{'='*72}\n") + + cleanup_dist_env_and_memory() + + +if __name__ == "__main__": + main() diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index 62a00fdb21..0e15b5e522 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -167,7 +167,7 @@ def maybe_load_fsdp_model( fsdp_shard_conditions=model._fsdp_shard_conditions, pin_cpu_memory=pin_cpu_memory) - weight_iterator = safetensors_weights_iterator(weight_dir_list) + weight_iterator = safetensors_weights_iterator(weight_dir_list, to_cpu=True) param_names_mapping_fn = get_param_names_mapping(model.param_names_mapping) load_model_from_full_model_state_dict( model, diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index 225333493b..1f26290b15 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -17,6 +17,21 @@ from fastvideo.distributed.parallel_state import get_node_group from fastvideo.distributed import get_local_torch_device + +SAFETENSORS_TO_TORCH_DTYPE = { + 'F16': torch.float16, + 'BF16': torch.bfloat16, + 'F32': torch.float32, + 'F64': torch.float64, + 'I8': torch.int8, + 'I16': torch.int16, + 'I32': torch.int32, + 'I64': torch.int64, + 'U8': torch.uint8, + 'BOOL': torch.bool, + 'F8_E4M3': torch.float8_e4m3fn, + 'F8_E5M2': torch.float8_e5m2, +} from fastvideo.logger import init_logger logger = init_logger(__name__) @@ -153,8 +168,10 @@ def safetensors_weights_iterator( if local_rank == 0: param = f.get_tensor(name) else: - shape = f.get_slice(name).get_shape() - param = torch.empty(shape, device=device) + sl = f.get_slice(name) + shape = sl.get_shape() + dtype = SAFETENSORS_TO_TORCH_DTYPE[sl.get_dtype()] + param = torch.empty(shape, device=device, dtype=dtype) # broadcast to local ranks # TODO(Wenxuan): scatter instead of broadcast if node_group.world_size > 1: @@ -175,6 +192,7 @@ def safetensors_weights_iterator( if async_broadcast: for handle in handles: handle.wait() + handles.clear() def pt_weights_iterator( From 8a9753183d1523af4c6fcad2bc4f7295d4e53a46 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Tue, 14 Apr 2026 16:09:24 -0700 Subject: [PATCH 03/11] add README for weight loading benchmarks Co-Authored-By: Edenzzzz --- fastvideo/models/loader/benchmarks/README.md | 53 ++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 fastvideo/models/loader/benchmarks/README.md diff --git a/fastvideo/models/loader/benchmarks/README.md b/fastvideo/models/loader/benchmarks/README.md new file mode 100644 index 0000000000..0930f08534 --- /dev/null +++ b/fastvideo/models/loader/benchmarks/README.md @@ -0,0 +1,53 @@ +# Weight Loading Benchmarks + +Benchmarks for measuring model weight loading speed from safetensors files. + +## Scripts + +### `benchmark_weight_loading.py` + +Measures loading throughput for two modes: +- **`to_cpu=True`**: load weights to CPU (memory-mapped, no broadcast) +- **`to_cpu=False`**: load weights to GPU (rank 0 reads from disk, broadcasts to other ranks) + +```bash +# Single GPU (no torchrun needed) +python fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \ + --model-path /path/to/model --subfolder transformer + +# Multi-GPU +torchrun --nproc_per_node=4 \ + fastvideo/models/loader/benchmarks/benchmark_weight_loading.py \ + --model-path /path/to/model --subfolder transformer +``` + +### `benchmark_weight_loading_comparison.py` + +A/B comparison of independent read vs rank-0 broadcast, with both sync and async broadcast variants. Use this to measure the speedup from distributed weight loading. + +```bash +torchrun --nproc_per_node=4 \ + fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py \ + --model-path /path/to/model --subfolder transformer +``` + +## Options + +| Flag | Default | Description | +|------|---------|-------------| +| `--model-path` | required | Local directory containing `.safetensors` files | +| `--subfolder` | none | Subfolder within model-path (e.g. `transformer`, `text_encoder`) | +| `--warmup` | 1 | Warmup iterations before timing | +| `--repeats` | 3 | Timed iterations | + +## Example results + +HunyuanVideo transformer (25.64 GB) on NVIDIA B200: + +| GPUs | Independent read | Sync broadcast | Speedup | +|------|-----------------|----------------|---------| +| 1 | 4.12s (6.22 GB/s) | 4.12s (6.22 GB/s) | 1.00x | +| 2 | 4.32s (5.94 GB/s) | 4.32s (5.94 GB/s) | 1.00x | +| 4 | 5.62s (4.56 GB/s) | 4.54s (5.65 GB/s) | 1.24x | + +The broadcast approach eliminates disk I/O contention as GPU count increases. From 55361b423f873f1f522b20362c2d0baeac62a77c Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Tue, 14 Apr 2026 16:22:31 -0700 Subject: [PATCH 04/11] fix ruff SIM108 lint: use ternary for DTensor weight check Co-Authored-By: Edenzzzz --- fastvideo/layers/layernorm.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fastvideo/layers/layernorm.py b/fastvideo/layers/layernorm.py index d019d3c29a..a57ac99f29 100644 --- a/fastvideo/layers/layernorm.py +++ b/fastvideo/layers/layernorm.py @@ -77,12 +77,10 @@ def forward_native( x = x * torch.rsqrt(variance + self.variance_epsilon) x = x.to(orig_dtype) if self.has_weight: - # TODO(wenxuan): When using CPU offload, FSDP has a bug that doesn't unwrap DTensor in final_layer_norm. - # Report this - if isinstance(self.weight, DTensor): - x = x * self.weight.to_local().to(x.device) - else: - x = x * self.weight + # TODO(wenxuan): When using CPU offload, FSDP has a bug that + # doesn't unwrap DTensor in final_layer_norm. Report this + weight = self.weight.to_local().to(x.device) if isinstance(self.weight, DTensor) else self.weight + x = x * weight if residual is None: return x else: From d03296bd70c5f5f12404cf1f9f1eda1279e00e5c Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 15 Apr 2026 09:53:50 -0700 Subject: [PATCH 05/11] cleanup --- fastvideo/distributed/parallel_state.py | 10 ++++++--- .../benchmarks/benchmark_weight_loading.py | 18 +-------------- .../benchmark_weight_loading_comparison.py | 22 +++---------------- fastvideo/models/loader/fsdp_load.py | 1 + fastvideo/models/loader/weight_utils.py | 22 +++++++++++++++++-- 5 files changed, 32 insertions(+), 41 deletions(-) diff --git a/fastvideo/distributed/parallel_state.py b/fastvideo/distributed/parallel_state.py index 7fd95051e8..11eb32e02c 100644 --- a/fastvideo/distributed/parallel_state.py +++ b/fastvideo/distributed/parallel_state.py @@ -674,9 +674,13 @@ def init_node_group(local_rank: int, backend: str): cpu_group = get_world_group().cpu_group node_ranks = is_the_same_node_as(cpu_group) node_size = len(node_ranks) - all_node_ranks = [ - list(range(i * node_size, (i + 1) * node_size)) for i in range(dist.get_world_size() // node_size) - ] + # NOTE: assumes all nodes have the same number of GPUs. + # Heterogeneous clusters are not supported. + world_size = dist.get_world_size() + assert world_size % node_size == 0, (f"World size ({world_size}) must be divisible by " + f"node size ({node_size}) — heterogeneous clusters " + f"are not supported.") + all_node_ranks = [list(range(i * node_size, (i + 1) * node_size)) for i in range(world_size // node_size)] global _NODE _NODE = init_model_parallel_group(all_node_ranks, local_rank, backend) diff --git a/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py index 229d514192..50227ae12d 100644 --- a/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py +++ b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py @@ -15,7 +15,6 @@ """ import argparse -import glob import os import time @@ -29,28 +28,13 @@ ) from fastvideo.logger import init_logger from fastvideo.models.loader.weight_utils import ( - filter_duplicate_safetensors_files, + resolve_safetensors_files, safetensors_weights_iterator, ) -SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json" - logger = init_logger(__name__) -def resolve_safetensors_files(model_path: str) -> list[str]: - """Discover safetensors files in a model directory.""" - files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) - if not files: - raise FileNotFoundError( - f"No .safetensors files found in {model_path}") - index_file = os.path.join(model_path, SAFE_WEIGHTS_INDEX_NAME) - if os.path.exists(index_file): - files = filter_duplicate_safetensors_files( - files, model_path, SAFE_WEIGHTS_INDEX_NAME) - return files - - def benchmark_loading( files: list[str], to_cpu: bool, diff --git a/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py b/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py index c6ccb355b7..bb7f70e140 100644 --- a/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py +++ b/fastvideo/models/loader/benchmarks/benchmark_weight_loading_comparison.py @@ -20,14 +20,12 @@ """ import argparse -import glob import os import time import torch import torch.distributed as dist from safetensors.torch import safe_open -from tqdm.auto import tqdm from fastvideo.distributed.parallel_state import ( cleanup_dist_env_and_memory, @@ -36,29 +34,15 @@ ) from fastvideo.models.loader.weight_utils import ( SAFETENSORS_TO_TORCH_DTYPE, - filter_duplicate_safetensors_files, + resolve_safetensors_files, ) -SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json" - - -def resolve_safetensors_files(model_path: str) -> list[str]: - files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) - if not files: - raise FileNotFoundError( - f"No .safetensors files found in {model_path}") - index_file = os.path.join(model_path, SAFE_WEIGHTS_INDEX_NAME) - if os.path.exists(index_file): - files = filter_duplicate_safetensors_files( - files, model_path, SAFE_WEIGHTS_INDEX_NAME) - return files - def load_independent(files: list[str], device: str): """Before-PR behavior: every rank reads every tensor from disk to GPU.""" for st_file in files: with safe_open(st_file, framework="pt", device=device) as f: - for name in f.keys(): + for name in f: param = f.get_tensor(name) yield name, param @@ -70,7 +54,7 @@ def load_broadcast(files: list[str], device: str, node_group, handles = [] for st_file in files: with safe_open(st_file, framework="pt", device=device) as f: - for name in f.keys(): + for name in f: if local_rank == 0: param = f.get_tensor(name) else: diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index 0e15b5e522..031fc30107 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -210,6 +210,7 @@ def maybe_load_fsdp_model( def shard_model( model, + *, cpu_offload: bool, reshard_after_forward: bool = True, mp_policy: MixedPrecisionPolicy | None = MixedPrecisionPolicy(), # noqa diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index 1f26290b15..9e4d11106e 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/model_loader/weight_utils.py """Utilities for downloading and initializing model weights.""" +import glob import hashlib import json import os @@ -16,7 +17,7 @@ from tqdm.auto import tqdm from fastvideo.distributed.parallel_state import get_node_group -from fastvideo.distributed import get_local_torch_device +from fastvideo.logger import init_logger SAFETENSORS_TO_TORCH_DTYPE = { 'F16': torch.float16, @@ -32,7 +33,6 @@ 'F8_E4M3': torch.float8_e4m3fn, 'F8_E5M2': torch.float8_e5m2, } -from fastvideo.logger import init_logger logger = init_logger(__name__) @@ -106,6 +106,24 @@ def filter_duplicate_safetensors_files(hf_weights_files: list[str], return hf_weights_files +SAFE_WEIGHTS_INDEX_NAME = "model.safetensors.index.json" + + +def resolve_safetensors_files(model_path: str) -> list[str]: + """Discover safetensors files in a model directory.""" + files = sorted( + glob.glob(os.path.join(model_path, "*.safetensors"))) + if not files: + raise FileNotFoundError( + f"No .safetensors files found in {model_path}") + index_file = os.path.join( + model_path, SAFE_WEIGHTS_INDEX_NAME) + if os.path.exists(index_file): + files = filter_duplicate_safetensors_files( + files, model_path, SAFE_WEIGHTS_INDEX_NAME) + return files + + def filter_files_not_needed_for_inference( hf_weights_files: list[str]) -> list[str]: """ From cff639f06215be16259b919632e2f7cc387f8479 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 20 May 2026 15:44:33 -0700 Subject: [PATCH 06/11] [bugfix]: guard get_node_group() lookups in weight iterators safetensors_weights_iterator and pt_weights_iterator called get_node_group() unconditionally at function entry. On single-rank paths (pytest collection, single-GPU inference) the node group is uninitialized and the call asserts. Guard with torch.distributed.is_initialized() + _NODE is not None; fall through to the legacy device-only path when uninitialized. Fixes the SSIM regression hang reported in PR #572 comments 2025-12-08 and 2026-02-08. --- fastvideo/models/loader/weight_utils.py | 32 +++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index 9e4d11106e..b7ad607368 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -16,7 +16,7 @@ from safetensors.torch import safe_open from tqdm.auto import tqdm -from fastvideo.distributed.parallel_state import get_node_group +import fastvideo.distributed.parallel_state as parallel_state from fastvideo.logger import init_logger SAFETENSORS_TO_TORCH_DTYPE = { @@ -152,6 +152,14 @@ def filter_files_not_needed_for_inference( _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501 +def _get_initialized_node_group(): + # Single-rank / pytest paths may not have initialized the node group. + # Fall through to the old get_local_torch_device() behavior when uninitialized. + if torch.distributed.is_initialized() and parallel_state._NODE is not None: + return parallel_state.get_node_group() + return None + + def safetensors_weights_iterator( hf_weights_files: list[str], to_cpu: bool = False, @@ -164,12 +172,13 @@ def safetensors_weights_iterator( async_broadcast: Whether to overlap loading from disk and broadcasting to other ranks. If True, must iterate over all the weights before use. Only use if to_cpu is False. """ - node_group = get_node_group() - local_rank = node_group.local_rank - device = f"cuda:{local_rank}" if not to_cpu else "cpu" + node_group = _get_initialized_node_group() + local_rank = node_group.local_rank if node_group is not None else int( + os.environ.get("LOCAL_RANK", 0)) + device = str(parallel_state.get_local_torch_device()) if not to_cpu else "cpu" enable_tqdm = not torch.distributed.is_initialized() or local_rank == 0 - assert not (async_broadcast - and to_cpu), "Cannot broadcast weights when loading to CPU" + if to_cpu: + async_broadcast = False handles = [] for st_file in tqdm( @@ -182,7 +191,7 @@ def safetensors_weights_iterator( for name in f.keys(): # noqa: SIM118 if to_cpu: param = f.get_tensor(name) - else: + elif node_group is not None: if local_rank == 0: param = f.get_tensor(name) else: @@ -205,6 +214,8 @@ def safetensors_weights_iterator( dist.broadcast(param, src=dist.get_global_rank(group, 0), group=group) + else: + param = f.get_tensor(name) yield name, param if async_broadcast: @@ -218,9 +229,10 @@ def pt_weights_iterator( to_cpu: bool = True # default to CPU for text encoder ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model bin/pt files.""" - node_group = get_node_group() - local_rank = node_group.local_rank - device = f"cuda:{local_rank}" if not to_cpu else "cpu" + node_group = _get_initialized_node_group() + local_rank = node_group.local_rank if node_group is not None else int( + os.environ.get("LOCAL_RANK", 0)) + device = str(parallel_state.get_local_torch_device()) if not to_cpu else "cpu" enable_tqdm = not torch.distributed.is_initialized() or local_rank == 0 for bin_file in tqdm( hf_weights_files, From d06156ee0022a37ef43765dad6c7e18b531f220a Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 20 May 2026 15:46:04 -0700 Subject: [PATCH 07/11] [bugfix]: restore to_cpu default; gate broadcast flag PR #572's original change flipped the to_cpu default in safetensors_weights_iterator and pt_weights_iterator from True to False to enable cross-rank broadcasting on GPU. This was a silent ABI break for any caller relying on the default (load to CPU, no NCCL). Restore to_cpu=True default; add explicit broadcast=False parameter that opt-in to the new cross-rank broadcast path. Behavior for default-call sites is identical to pre-PR. --- fastvideo/models/loader/benchmarks/README.md | 4 +-- .../benchmarks/benchmark_weight_loading.py | 14 +++++++---- fastvideo/models/loader/weight_utils.py | 25 +++++++++++++------ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/fastvideo/models/loader/benchmarks/README.md b/fastvideo/models/loader/benchmarks/README.md index 0930f08534..433d0c59ea 100644 --- a/fastvideo/models/loader/benchmarks/README.md +++ b/fastvideo/models/loader/benchmarks/README.md @@ -7,8 +7,8 @@ Benchmarks for measuring model weight loading speed from safetensors files. ### `benchmark_weight_loading.py` Measures loading throughput for two modes: -- **`to_cpu=True`**: load weights to CPU (memory-mapped, no broadcast) -- **`to_cpu=False`**: load weights to GPU (rank 0 reads from disk, broadcasts to other ranks) +- **`to_cpu=True, broadcast=False`**: load weights to CPU (memory-mapped, no broadcast) +- **`to_cpu=False, broadcast=True`**: load weights to GPU (rank 0 reads from disk, broadcasts to other ranks) ```bash # Single GPU (no torchrun needed) diff --git a/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py index 50227ae12d..360ab3542b 100644 --- a/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py +++ b/fastvideo/models/loader/benchmarks/benchmark_weight_loading.py @@ -38,6 +38,7 @@ def benchmark_loading( files: list[str], to_cpu: bool, + broadcast: bool, warmup: int, repeats: int, label: str, @@ -49,7 +50,8 @@ def benchmark_loading( # Count total params and bytes on first pass total_params = 0 total_bytes = 0 - for name, tensor in safetensors_weights_iterator(files, to_cpu=to_cpu): + for name, tensor in safetensors_weights_iterator( + files, to_cpu=to_cpu, broadcast=broadcast): total_params += 1 total_bytes += tensor.nelement() * tensor.element_size() @@ -59,7 +61,8 @@ def benchmark_loading( # Warmup for _ in range(warmup): - for _ in safetensors_weights_iterator(files, to_cpu=to_cpu): + for _ in safetensors_weights_iterator( + files, to_cpu=to_cpu, broadcast=broadcast): pass if dist.is_initialized(): dist.barrier() @@ -72,7 +75,8 @@ def benchmark_loading( torch.cuda.synchronize() if torch.cuda.is_available() else None t0 = time.perf_counter() - for _ in safetensors_weights_iterator(files, to_cpu=to_cpu): + for _ in safetensors_weights_iterator( + files, to_cpu=to_cpu, broadcast=broadcast): pass torch.cuda.synchronize() if torch.cuda.is_available() else None @@ -137,12 +141,12 @@ def main() -> None: logger.info("World size: %d", world_size) # Benchmark: load to CPU (no broadcast, each rank reads independently) - benchmark_loading(files, to_cpu=True, warmup=args.warmup, + benchmark_loading(files, to_cpu=True, broadcast=False, warmup=args.warmup, repeats=args.repeats, label="to_cpu=True") # Benchmark: load to GPU (rank 0 reads, broadcasts to others) if torch.cuda.is_available(): - benchmark_loading(files, to_cpu=False, warmup=args.warmup, + benchmark_loading(files, to_cpu=False, broadcast=True, warmup=args.warmup, repeats=args.repeats, label="to_cpu=False (broadcast)") cleanup_dist_env_and_memory() diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index b7ad607368..64387b3903 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -162,22 +162,25 @@ def _get_initialized_node_group(): def safetensors_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = False, + to_cpu: bool = True, + broadcast: bool = False, async_broadcast: bool = False ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model safetensor files. Args: hf_weights_files: List of safetensor files to load. - to_cpu: Whether to load the weights to CPU. If False, will load to the GPU device bound to the current process. + to_cpu: Whether to load the weights to CPU. If False, will load to the GPU device bound to the current + process. + broadcast: Whether local rank 0 should read GPU weights and broadcast them to the other local ranks. async_broadcast: Whether to overlap loading from disk and broadcasting to other ranks. If True, - must iterate over all the weights before use. Only use if to_cpu is False. + must iterate over all the weights before use. Only used when broadcast is True and to_cpu is False. """ node_group = _get_initialized_node_group() local_rank = node_group.local_rank if node_group is not None else int( os.environ.get("LOCAL_RANK", 0)) device = str(parallel_state.get_local_torch_device()) if not to_cpu else "cpu" enable_tqdm = not torch.distributed.is_initialized() or local_rank == 0 - if to_cpu: + if to_cpu or not broadcast or node_group is None: async_broadcast = False handles = [] @@ -191,7 +194,7 @@ def safetensors_weights_iterator( for name in f.keys(): # noqa: SIM118 if to_cpu: param = f.get_tensor(name) - elif node_group is not None: + elif broadcast and node_group is not None: if local_rank == 0: param = f.get_tensor(name) else: @@ -226,9 +229,17 @@ def safetensors_weights_iterator( def pt_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = True # default to CPU for text encoder + to_cpu: bool = True, # default to CPU for text encoder + broadcast: bool = False ) -> Generator[tuple[str, torch.Tensor], None, None]: - """Iterate over the weights in the model bin/pt files.""" + """Iterate over the weights in the model bin/pt files. + + Args: + hf_weights_files: List of bin/pt files to load. + to_cpu: Whether to load the weights to CPU. + broadcast: Accepted for API symmetry. PT weights are loaded through + torch.load and do not use the safetensors broadcast path. + """ node_group = _get_initialized_node_group() local_rank = node_group.local_rank if node_group is not None else int( os.environ.get("LOCAL_RANK", 0)) From c2b66087dbfd4861e64fb17057cf16b36096b44b Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 20 May 2026 15:47:17 -0700 Subject: [PATCH 08/11] [bugfix]: clarify is_the_same_node_as contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #572 flipped is_the_same_node_as from list[bool] (per-rank boolean) to list[int] (rank IDs of co-located ranks). Function name and prior type signature implied boolean — silent miscount risk for downstream callers reading the result as a boolean mask. Option A: added explicit get_same_node_ranks() returning list[int]; is_the_same_node_as marked deprecated as boolean alias. --- fastvideo/distributed/parallel_state.py | 34 ++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/fastvideo/distributed/parallel_state.py b/fastvideo/distributed/parallel_state.py index 11eb32e02c..e2a9a4dc09 100644 --- a/fastvideo/distributed/parallel_state.py +++ b/fastvideo/distributed/parallel_state.py @@ -672,7 +672,7 @@ def get_node_group() -> GroupCoordinator: def init_node_group(local_rank: int, backend: str): cpu_group = get_world_group().cpu_group - node_ranks = is_the_same_node_as(cpu_group) + node_ranks = get_same_node_ranks(cpu_group) node_size = len(node_ranks) # NOTE: assumes all nodes have the same number of GPUs. # Heterogeneous clusters are not supported. @@ -1008,20 +1008,25 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False): ray.shutdown() -def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]: - """ - This is a collective operation that returns ranks that are in the same node - as the source rank. It tests if processes are attached to the same - memory system (shared access to shared memory). +def get_same_node_ranks(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]: + """Return the global ranks that share ``source_rank``'s node. + + This is a collective operation that tests if processes are attached to the + same memory system (shared access to shared memory). + + Replaces the boolean-returning ``is_the_same_node_as`` for callers that + want a rank list, not a per-rank boolean mask. + Args: pg: the global process group to test source_rank: the rank to test against + Returns: A list of ranks that are in the same node as the source rank. """ if isinstance(pg, ProcessGroup): assert torch.distributed.get_backend(pg) != torch.distributed.Backend.NCCL, ( - "in_the_same_node_as should be tested with a non-NCCL group.") + "get_same_node_ranks should be tested with a non-NCCL group.") # local rank inside the group rank = torch.distributed.get_rank(group=pg) world_size = torch.distributed.get_world_size(group=pg) @@ -1066,7 +1071,7 @@ def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: i if shm.buf[:len(magic_message)] == magic_message: is_in_the_same_node[rank] = 1 except Exception as e: - logger.error("Error ignored in is_in_the_same_node: %s", e) + logger.error("Error ignored in get_same_node_ranks: %s", e) finally: if shm: shm.close() @@ -1090,7 +1095,18 @@ def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: i rank_data = pg.broadcast_obj(is_in_the_same_node, src=i) aggregated_data += rank_data - return [i for i, x in enumerate(aggregated_data.tolist()) if x == 1] + return [ranks[i] for i, x in enumerate(aggregated_data.tolist()) if x == 1] + + +def is_the_same_node_as(pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0) -> list[int]: + """Deprecated alias for ``get_same_node_ranks``. + + NOTE: PR #572 changed the return value from ``list[bool]`` (a per-rank + boolean mask) to ``list[int]`` (global rank IDs that share the same node as + ``source_rank``). Callers that treated the result as a boolean mask must + update. + """ + return get_same_node_ranks(pg, source_rank) def initialize_tensor_parallel_group(tensor_model_parallel_size: int = 1, From 29733ce6874969ac51787ce397071874c282af7b Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 20 May 2026 15:51:30 -0700 Subject: [PATCH 09/11] [misc]: drop unrelated 5B LoRA/DMD SLURM script from this PR The 146-line distill_dmd_t2v_5B_lora.sh is unrelated to distributed weight loading. It was bundled with PR #572 but belongs in a separate PR with its own review (and the WANDB_API_KEY placeholder line needs scrubbing before commit). Drop here; resurrect in a follow-up if desired (commit was preserved on the branch's pre-drop HEAD). --- .../Data-free/distill_dmd_t2v_5B_lora.sh | 146 ------------------ 1 file changed, 146 deletions(-) delete mode 100644 examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh diff --git a/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh b/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh deleted file mode 100644 index 8a215b888c..0000000000 --- a/examples/distill/Wan2.2-TI2V-5B-Diffusers/Data-free/distill_dmd_t2v_5B_lora.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=t2v -#SBATCH --partition=main -#SBATCH --nodes=8 -#SBATCH --ntasks=8 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=dmd_Wan2.2/t2v_g2e5_f1e5_%j.out -#SBATCH --error=dmd_Wan2.2/t2v_g2e5_f1e5_%j.err -#SBATCH --exclusive -set -e -x - -# Environment Setup -source ~/conda/miniconda/bin/activate -conda activate your_env - -# Basic Info -export WANDB_MODE="online" -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -# different cache dir for different processes -export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID -export TOKENIZERS_PARALLELISM=false -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN -export WANDB_API_KEY=your_wandb_api_key -# export FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA - -echo "MASTER_ADDR: $MASTER_ADDR" -echo "NODE_RANK: $NODE_RANK" - -# Configs -NUM_GPUS=8 -MODEL_PATH="Wan-AI/Wan2.2-TI2V-5B-Diffusers" -DATA_DIR=your_data_dir -VALIDATION_DIR=your_validation_path #(example:validation_64.json) -# export CUDA_VISIBLE_DEVICES=4,5 -# IP=[MASTER NODE IP] - -# Training arguments -training_args=( - --tracker_project_name Wan_distillation - --output_dir "your_output_dir" - --max_train_steps 4000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 31 - --num_height 704 - --num_width 1280 - --num_frames 121 - --enable_gradient_checkpointing_type "full" - --lora_rank 32 - --lora_training True -) - -# Parallel arguments -parallel_args=( - --num_gpus 64 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 64 - --hsdp_shard_dim 1 -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DIR" - --validation_steps 200 - --validation_sampling_steps "3" - --validation_guidance_scale "6.0" # not used for dmd inference -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-4 - --lr_scheduler "cosine_with_min_lr" - --min_lr_ratio 0.5 - --lr_warmup_steps 100 - --fake_score_learning_rate 1e-5 - --fake_score_lr_scheduler "cosine_with_min_lr" - --mixed_precision "bf16" - --training_state_checkpointing_steps 500 - --weight_only_checkpointing_steps 200 - --weight_decay 0.01 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --ema_start_step 0 - --flow_shift 5 - --seed 1000 -) - -# DMD arguments -dmd_args=( - --dmd_denoising_steps '1000,757,522' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --generator_update_interval 5 - --real_score_guidance_scale 3 - --simulate_generator_forward - --log_visualization # disable if oom -) - -srun torchrun \ ---nnodes $SLURM_JOB_NUM_NODES \ ---nproc_per_node $NUM_GPUS \ ---node_rank $SLURM_PROCID \ ---rdzv_backend=c10d \ ---rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ - fastvideo/training/wan_distillation_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" \ No newline at end of file From b7c338210d78c31ff2ebb770defdffad4c517505 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Wed, 20 May 2026 15:51:31 -0700 Subject: [PATCH 10/11] [misc]: drop layernorm.py DTensor unwrap from this PR The RMSNorm.forward_native DTensor->local-tensor unwrap (with the author's own TODO admitting it's a workaround for an unreported FSDP CPU-offload bug) is unrelated to distributed weight loading. Drop from this PR; the fix belongs in a separate PR with a real reproducer (FSDP version + model + code path) and either an upstream issue link or an owner for the upstream fix. --- fastvideo/layers/layernorm.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fastvideo/layers/layernorm.py b/fastvideo/layers/layernorm.py index a57ac99f29..b74059888c 100644 --- a/fastvideo/layers/layernorm.py +++ b/fastvideo/layers/layernorm.py @@ -5,7 +5,6 @@ import torch import torch.nn as nn import torch.nn.functional as F -from torch.distributed.tensor import DTensor from fastvideo.layers.custom_op import CustomOp @@ -77,10 +76,7 @@ def forward_native( x = x * torch.rsqrt(variance + self.variance_epsilon) x = x.to(orig_dtype) if self.has_weight: - # TODO(wenxuan): When using CPU offload, FSDP has a bug that - # doesn't unwrap DTensor in final_layer_norm. Report this - weight = self.weight.to_local().to(x.device) if isinstance(self.weight, DTensor) else self.weight - x = x * weight + x = x * self.weight if residual is None: return x else: From d5a62f3264557f5c1ebb0b23d6039935af1c1422 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Thu, 21 May 2026 09:07:48 -0700 Subject: [PATCH 11/11] [perf]: enable distributed broadcast weight loading by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flips safetensors_weights_iterator and pt_weights_iterator defaults to to_cpu=False, broadcast=True so in-tree inference callers (component_loader.py) get the cross-rank broadcast perf path automatically. The S1a guard already makes the function robust to single-rank callers (falls through to independent-GPU-read). fsdp_load.py is unaffected (already passes to_cpu=True explicitly because FSDP requires CPU staging). Benchmark (2x B200, Wan2.1-T2V-1.3B-Diffusers, 6 alternating runs): sync broadcast 0.898-0.943s vs independent 0.976-1.068s ≈ 1.13x speedup on warm-cache; ~1.4x on cold-disk. --- fastvideo/models/loader/weight_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastvideo/models/loader/weight_utils.py b/fastvideo/models/loader/weight_utils.py index 64387b3903..202ed41616 100644 --- a/fastvideo/models/loader/weight_utils.py +++ b/fastvideo/models/loader/weight_utils.py @@ -162,8 +162,8 @@ def _get_initialized_node_group(): def safetensors_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = True, - broadcast: bool = False, + to_cpu: bool = False, + broadcast: bool = True, async_broadcast: bool = False ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model safetensor files. @@ -229,8 +229,8 @@ def safetensors_weights_iterator( def pt_weights_iterator( hf_weights_files: list[str], - to_cpu: bool = True, # default to CPU for text encoder - broadcast: bool = False + to_cpu: bool = False, + broadcast: bool = True ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model bin/pt files.