Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 158 additions & 9 deletions atom/model_loader/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import os
import logging
import re
import threading
import time
from glob import glob
from typing import Generator, Tuple
Expand Down Expand Up @@ -401,11 +402,135 @@ def extract_expert_target_and_id(name: str) -> Tuple[str, int] | None:
# rewritten name doesn't correspond to any model param. (orig, mapped) pairs.
dropped_ckpt_keys: list[tuple[str, str]] = []

with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
use_threadpool = envs.ATOM_LOADER_USE_THREADPOOL
if use_threadpool:
executor = concurrent.futures.ThreadPoolExecutor()
staging_map: dict = {} # id(param) -> entry, one per in-flight fused param
fallback_pids: set = set() # params that opted out of batching
staging_lock = threading.Lock()

moe_module_cache: dict = {}
param_batchable: dict = {}

def _lookup_moe_module(full_param_name: str):
module_path = full_param_name.rsplit(".", 1)[0]
if module_path not in moe_module_cache:
moe_module_cache[module_path] = (
model.get_submodule(module_path) if "." in full_param_name else None
)
return moe_module_cache[module_path]

def _param_is_batchable(param, full_param_name: str) -> bool:
pid = id(param)
if pid not in param_batchable:
moe = _lookup_moe_module(full_param_name)
expected = (
moe.expected_batched_arrivals(param)
if moe is not None and hasattr(moe, "stage_expert_weight")
else None
)
param_batchable[pid] = bool(expected)
return param_batchable[pid]

def _do_flush(param, staging):
if staging.dtype != param.data.dtype:
param.data.view(torch.uint8).copy_(staging)
else:
param.data.copy_(staging)

def _make_staging(param):
pin = torch.cuda.is_available()

def _alloc(pinned):
try:
t = torch.empty(
param.data.shape,
dtype=param.data.dtype,
device="cpu",
pin_memory=pinned,
)
t.zero_()
except NotImplementedError:
t = torch.empty(
param.data.shape,
dtype=torch.uint8,
device="cpu",
pin_memory=pinned,
)
t.zero_()
return t

try:
return _alloc(pin)
except RuntimeError as e:
logger.warning("Pinned staging alloc failed (%s); using unpinned.", e)
return _alloc(False)

def _fallback(param, full_param_name, shard_id, global_expert_id, loaded_weight):
param.weight_loader(
param, loaded_weight, full_param_name, shard_id, global_expert_id
)

def _stage_task(param, full_param_name, shard_id, global_expert_id, loaded_weight):
pid = id(param)
with staging_lock:
opted_out = pid in fallback_pids
entry = None if opted_out else staging_map.get(pid)
if opted_out:
_fallback(param, full_param_name, shard_id, global_expert_id, loaded_weight)
return

if entry is None:
moe = _lookup_moe_module(full_param_name)
new_entry = {
"staging": _make_staging(param),
"arrived": 0,
"expected": moe.expected_batched_arrivals(param),
"moe": moe,
"param": param,
"lock": threading.Lock(),
}
with staging_lock:
opted_out = pid in fallback_pids
if not opted_out:
entry = staging_map.get(pid)
if entry is None:
entry = staging_map[pid] = new_entry
if opted_out:
_fallback(
param, full_param_name, shard_id, global_expert_id, loaded_weight
)
return

moe = entry["moe"]
local_eid = moe._map_global_expert_id_to_local_expert_id(global_expert_id)
if local_eid == -1:
return

ok = moe.stage_expert_weight(
param=param,
staging=entry["staging"],
loaded_weight=loaded_weight,
local_expert_id=local_eid,
shard_id=shard_id,
weight_name=full_param_name,
)
if not ok:
with staging_lock:
fallback_pids.add(pid)
staging_map.pop(pid, None)
_fallback(param, full_param_name, shard_id, global_expert_id, loaded_weight)
return

with entry["lock"]:
entry["arrived"] += 1
flush_now = entry["arrived"] >= entry["expected"]
if flush_now:
_do_flush(param, entry["staging"])
with staging_lock:
if staging_map.get(pid) is entry:
del staging_map[pid]

num_threads = envs.ATOM_LOADER_NUM_THREADS
if num_threads > 1:
executor = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads)
else:
executor = None
futures = []
Expand Down Expand Up @@ -593,13 +718,24 @@ def _submit(fn, *args):
if "mtp" in name and not spec_decode:
matched = True
break
try:
param = model.get_parameter(name)
except AttributeError:
param = params_dict.get(name)
if param is None:
# Parameter absent from model (e.g. weight scales for
# an unquantized drafter MTP block); skip silently.
matched = True
break
if executor is not None and _param_is_batchable(param, name):
_submit(
_stage_task,
param,
name,
shard_id,
expert_id,
weight_tensor,
)
loaded_weights_record.add(prefix + name)
matched = True
break
weight_loader = getattr(param, "weight_loader")
_submit(
weight_loader,
Expand Down Expand Up @@ -657,9 +793,22 @@ def _submit(fn, *args):
)
_submit(weight_loader, param, weight_tensor)
loaded_weights_record.add(prefix + name)

if executor is not None:
# Drain all tasks (surfacing errors) before the safety flush.
for future in concurrent.futures.as_completed(futures):
future.result()

with staging_lock:
pending = list(staging_map.values())
staging_map.clear()
if pending:
raise RuntimeError(
f"Batched loader: {len(pending)} MoE param group(s) under-filled "
f"Set ATOM_LOADER_NUM_THREADS=1 to use the per-expert loader."
)
finally:
if executor is not None:
concurrent.futures.wait(futures)
executor.shutdown(wait=True)

# Verify every model parameter actually got loaded from the checkpoint.
Expand Down
175 changes: 136 additions & 39 deletions atom/model_ops/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3189,6 +3189,127 @@ def mxf4_merged_weight_loader(
narrow_weight = loaded_weight
target_param[: narrow_weight.shape[0]].copy_(narrow_weight)

def _copy_expert_shard(
self,
param: torch.nn.Parameter,
expert_data: torch.Tensor,
shard_id: str,
shard_dim: int,
loaded_weight: torch.Tensor,
weight_name: str,
) -> bool:
"""Copy one expert's shard into ``expert_data``. Shared by
``weight_loader`` (GPU dest) and ``stage_expert_weight`` (CPU staging
dest) so the two paths can't drift. Returns False for cases the batched
path can't handle (per-Tensor scale, ``weight_shape``); caller falls back.
"""
# scales / zero-points / offset (group + per-channel)
if "scale" in weight_name or "zero" in weight_name or "offset" in weight_name:
quant_method = self.layer_quant_config.quant_type
if quant_method == QuantType.per_Token:
self._load_per_channel_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
)
return True
if quant_method in (QuantType.per_1x128, QuantType.per_1x32):
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
load_full=getattr(param, "load_full_w2", False),
)
return True
return False

# model weights; `weight_shape` shares the substring but isn't batchable
if "weight" in weight_name and "weight_shape" not in weight_name:
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
)
return True

return False

def stage_expert_weight(
self,
param: torch.nn.Parameter,
staging: torch.Tensor,
loaded_weight: torch.Tensor,
local_expert_id: int,
shard_id: str,
weight_name: str,
) -> bool:
# Cases not handled by the batched path — caller falls back.
if (
"input_scale" in weight_name
or "g_idx" in weight_name
or "weight_shape" in weight_name
or weight_name == ""
):
return False
if shard_id not in ("w1", "w2", "w3"):
return False

# compressed-tensors packed-weight flip (matches weight_loader)
if self.quant_method.__class__.__name__ in (
"CompressedTensorsWNA16MarlinMoEMethod",
"CompressedTensorsWNA16MoEMethod",
):
loaded_weight = loaded_weight.t().contiguous()

SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
is_transposed = getattr(param, "is_transposed", False)
shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
if is_transposed:
shard_dim = int(not shard_dim)

if len(loaded_weight.shape) == 3:
return False

expert_data = staging[local_expert_id]

if (
staging.dtype == torch.uint8
and param.data.dtype == dtypes.fp4x2
and loaded_weight.dtype == dtypes.fp4x2
):
loaded_weight = loaded_weight.view(torch.uint8)

return self._copy_expert_shard(
param=param,
expert_data=expert_data,
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
weight_name=weight_name,
)

def expected_batched_arrivals(self, param: torch.nn.Parameter) -> Optional[int]:
w13_batchable = [getattr(self, "w13_weight", None)]
w2_batchable = [getattr(self, "w2_weight", None)]
if self.layer_quant_config.quant_type in (
QuantType.per_Token,
QuantType.per_1x128,
QuantType.per_1x32,
):
w13_batchable.append(getattr(self, "w13_weight_scale", None))
w2_batchable.append(getattr(self, "w2_weight_scale", None))
if any(param is p for p in w13_batchable if p is not None):
return self.local_num_experts * 2
if any(param is p for p in w2_batchable if p is not None):
return self.local_num_experts
return None

def weight_loader(
self,
param: torch.nn.Parameter,
Expand Down Expand Up @@ -3268,35 +3389,22 @@ def weight_loader(
)
return

# Case weight scales, zero_points and offset
# Group/per-channel scales and model weights share one copy path with
# the batched loader (_copy_expert_shard).
# TODO @dsikka: once hardened, refactor to use vLLM Parameters.
if self._copy_expert_shard(
param=param,
expert_data=expert_data,
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
weight_name=weight_name,
):
return

# Case per-Tensor weight scale
if "scale" in weight_name or "zero" in weight_name or "offset" in weight_name:
# load the weight scales and zp based on the quantization scheme
# supported weight scales/zp can be found in
# FusedMoeWeightScaleSupported
# TODO @dsikka: once hardened, refactor to use vLLM Parameters
# specific to each case
quant_method = self.layer_quant_config.quant_type
if quant_method == QuantType.per_Token:
self._load_per_channel_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
)
elif quant_method in [
QuantType.per_1x128,
QuantType.per_1x32,
]:
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
load_full=getattr(param, "load_full_w2", False),
)
elif quant_method == QuantType.per_Tensor:
if self.layer_quant_config.quant_type == QuantType.per_Tensor:
self._load_per_tensor_weight_scale(
shard_id=shard_id,
param=param,
Expand All @@ -3313,17 +3421,6 @@ def weight_loader(
)
return

# Case model weights
if "weight" in weight_name:
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.tp_rank,
)
return

@staticmethod
def select_experts(
hidden_states: torch.Tensor,
Expand Down
Loading
Loading