Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -421,27 +421,22 @@ def apply_weights(
activation_type=activation_type,
)
else:
quant_info = TritonMoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
use_fp8_w8a8=True,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
block_shape=self.weight_block_size,
)
quant_info = self.get_triton_quant_info(layer)
return self.runner.run(dispatch_output, quant_info)
else:
quant_info = TritonMoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
use_fp8_w8a8=True,
per_channel_quant=self.weight_quant.strategy
== QuantizationStrategy.CHANNEL,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
)
return self.runner.run(dispatch_output, quant_info)
return self.runner.run(dispatch_output, self.get_triton_quant_info(layer))

def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo:
# Shared with FusedMoEWithLoRA so both runners use the same base quantization.
return TritonMoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
use_fp8_w8a8=True,
per_channel_quant=self.weight_quant.strategy
== QuantizationStrategy.CHANNEL,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
block_shape=self.weight_block_size,
)
28 changes: 21 additions & 7 deletions python/sglang/srt/lora/lora.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,33 +141,42 @@ def _is_non_gated_moe_weight(self, weight_name: str) -> bool:
return self._moe_is_gated_by_layer.get(layer_id) is False

def initialize_weights(self):
from sglang.srt.lora.utils import get_normalized_target_modules

model_path = self.config.path
loader = DefaultModelLoader(self.load_config)
revision = getattr(self.config.hf_config, "revision", None)

# Get normalized target modules for filtering
normalized_target_modules = get_normalized_target_modules(
self.config.target_modules
)
for name, loaded_weight in loader._get_weights_iterator(
DefaultModelLoader.Source(
model_path, revision=revision, fall_back_to_pt=True
)
):
self._process_weight(name, loaded_weight)
self._process_weight(name, loaded_weight, normalized_target_modules)

self._normalize_weights()

def initialize_weights_from_tensors(self, tensors: Dict[str, torch.Tensor]):
for name, tensor in tensors.items():
self._process_weight(name, tensor)

self._normalize_weights()

def _process_weight(self, name: str, loaded_weight: torch.Tensor):
from sglang.srt.lora.utils import get_normalized_target_modules

normalized_target_modules = get_normalized_target_modules(
self.config.target_modules
)
for name, tensor in tensors.items():
self._process_weight(name, tensor, normalized_target_modules)

self._normalize_weights()

def _process_weight(
self,
name: str,
loaded_weight: torch.Tensor,
normalized_target_modules: set[str],
):
# Remap PEFT "unembed_tokens" key to "lm_head" so the weight is
# recognized and loaded into the correct buffer.
if "unembed_tokens" in name:
Expand Down Expand Up @@ -462,7 +471,12 @@ def _normalize_in_proj_qkvz(self, weights: Dict[str, torch.Tensor]):
def normalize_gate_up_proj(
self, weight_names: List[str], weights: Dict[str, torch.Tensor]
):
from sglang.srt.lora.utils import PRESERVED_LORA_TARGET_NAMES

for weight_name in weight_names:
if any(f".{name}." in weight_name for name in PRESERVED_LORA_TARGET_NAMES):
# The attention output gate has no up_proj to stack with.
continue
if "gate_proj" in weight_name:
up_name = weight_name.replace("gate_proj", "up_proj")
gate_up_name = weight_name.replace("gate_proj", "gate_up_proj")
Expand Down
30 changes: 27 additions & 3 deletions python/sglang/srt/lora/mem_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,21 @@ def _has_shared_fused_moe(base_model: torch.nn.Module) -> bool:
getattr(m, "is_shared_fused_moe", False) for m in base_model.modules()
)

@staticmethod
def _get_dense_module_layer_ids(
base_model: torch.nn.Module, module_name: str
) -> Set[int]:
"""Decoder layers containing this linear outside the fused experts."""
from sglang.srt.layers.linear import LinearBase

return {
layer_id
for name, module in base_model.named_modules()
if name.endswith(f".{module_name}")
and (layer_id := get_layer_id(name)) is not None
and isinstance(unwrap_lora_layer(module), LinearBase)
}

@staticmethod
def _has_moe_module(base_model: torch.nn.Module) -> bool:
# Config-only detection isn't reliable: some dense configs (e.g.
Expand Down Expand Up @@ -610,10 +625,15 @@ def init_buffers(self, base_model: torch.nn.Module):
# Cached once so the per-expert load path doesn't re-walk the HF
# config for every adapter.
self._num_experts_local: int = self._get_num_local_experts(base_model)
dense_layer_ids = {
name: self._get_dense_module_layer_ids(base_model, name)
for name in self.target_modules & {"gate_up_proj", "down_proj"}
}

def alloc_per_layer(
module_key: str,
get_lora_shape_fn: Callable[[str, torch.nn.Module, int, int], Tuple[int]],
layer_ids: Optional[Set[int]] = None,
) -> List[Optional[torch.Tensor]]:
return [
(
Expand All @@ -625,6 +645,7 @@ def alloc_per_layer(
device=device,
)
if idx in self.lora_layer_ids
and (layer_ids is None or idx in layer_ids)
else None
)
for idx in range(self.num_layer)
Expand All @@ -650,10 +671,13 @@ def init_buffer(
# Special handling for ambiguous target modules that can be in different contexts
ambiguous_modules = {"gate_up_proj", "down_proj"}
if module_name in ambiguous_modules and has_moe:
# Allocate shared expert version (3D) only when model has shared experts
if has_shared_experts:
# Hybrid models also need 3D buffers for their dense MLPs.
layer_ids = (
None if has_shared_experts else dense_layer_ids[module_name]
)
if layer_ids is None or layer_ids:
buffer[module_name] = alloc_per_layer(
module_name, get_lora_shape_fn
module_name, get_lora_shape_fn, layer_ids
)

# MoE expert version (4D)
Expand Down
17 changes: 16 additions & 1 deletion python/sglang/srt/lora/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,8 @@ def get_default_hidden_dim(
o_head_dim * config.num_attention_heads,
config.hidden_size,
)
elif module_name == "self_attn.gate_proj":
return config.hidden_size, config.num_attention_heads * head_dim
elif module_name == "gate_up_proj":
inter = config.intermediate_size
first_k = getattr(config, "first_k_dense_replace", None)
Expand Down Expand Up @@ -300,6 +302,9 @@ def get_normalized_target_modules(
Mapping a list of target module name to names of the normalized LoRA weights.
Handles both base module names (e.g., "gate_proj") and prefixed module names (e.g., "feed_forward.gate_proj").

The attention gate must be named "self_attn.gate_proj"; bare "gate_proj"
maps only to the MLP/MoE gate_up_proj.

Also handles PEFT shorthand strings like "all-linear" or "all" by returning
{"all"} as a sentinel value. Callers that need a concrete module set
should use :func:`auto_detect_lora_target_modules` to resolve the shorthand
Expand Down Expand Up @@ -340,6 +345,10 @@ def get_normalized_target_modules(

result = set()
for name in target_modules:
qualified_name = ".".join(name.split(".")[-2:])
if qualified_name in PRESERVED_LORA_TARGET_NAMES:
result.add(qualified_name)
continue
base_name = name.split(".")[-1]
normalized_name = params_mapping.get(base_name, base_name)
result.add(normalized_name)
Expand Down Expand Up @@ -401,6 +410,8 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s
DSA_INDEXER_LORA_NAMES = frozenset(
{"indexer.wq_b", "indexer.wk", "indexer.weights_proj"}
)
# Preserve the parent because the leaf alone names an MLP projection.
PRESERVED_LORA_TARGET_NAMES = frozenset({"self_attn.gate_proj"})
REPLICATED_LINEAR_LORA_NAMES = [
"fused_qkv_a_proj_with_mqa",
"fc1_latent_proj",
Expand All @@ -423,6 +434,7 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s
"wo_ud",
"in_proj",
"in_proj_qkvz",
"self_attn.gate_proj",
}
)

Expand All @@ -449,6 +461,7 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s
"kv_b_proj",
}
| DSA_INDEXER_LORA_NAMES
| PRESERVED_LORA_TARGET_NAMES
)


Expand Down Expand Up @@ -483,7 +496,9 @@ def auto_detect_lora_target_modules(model: "torch.nn.Module") -> set:
parts = name.split(".")
leaf_name = parts[-1]
parent_qualified_name = ".".join(parts[-2:])
if parent_qualified_name in DSA_INDEXER_LORA_NAMES:
if parent_qualified_name in (
DSA_INDEXER_LORA_NAMES | PRESERVED_LORA_TARGET_NAMES
):
raw_names.add(parent_qualified_name)
elif leaf_name in dsa_indexer_leaf_names:
# Bare DSA indexer leaf names are ambiguous across model
Expand Down
38 changes: 31 additions & 7 deletions python/sglang/srt/models/beam.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,9 +514,12 @@ def __init__(
self.sliding_window = config.sliding_window - 1 if self.apply_rope else -1
# MergedColumnParallelLinear shards each projection independently, but
# does not replicate KV heads. Keep the existing quantized/replicated
# paths until their loading and backend contracts are validated.
# paths until their loading and backend contracts are validated. LoRA
# wraps qkv_proj and gate_proj separately.
self.use_packed_projection = (
quant_config is None and config.num_key_value_heads >= self.tp_size
quant_config is None
and config.num_key_value_heads >= self.tp_size
and not (get_lora().enable_lora or get_lora().lora_paths)
)
if self.use_packed_projection:
self.qkv_gate_proj = MergedColumnParallelLinear(
Expand Down Expand Up @@ -944,9 +947,11 @@ def __init__(
)
# Decoder layers whose outputs feed a speculative drafter (DFlash).
self.layers_to_capture = ()
# Deferred expert sums need the BF16 Triton runner's per-expert outputs.
# Deferred expert sums read base weights directly, bypassing LoRA.
self.fuse_moe_all_reduce = (
quant_config is None and config.num_experts_per_tok == 8
quant_config is None
and config.num_experts_per_tok == 8
and not (get_lora().enable_lora or get_lora().lora_paths)
)
self._beam_all_reduce_comm = None
# Offload wrappers load weights through each layer's regular forward.
Expand Down Expand Up @@ -1286,14 +1291,27 @@ class BeamForCausalLM(nn.Module):
"gate_up_proj": ["gate_proj", "up_proj"],
}

# The attention output gate keeps its qualified name to distinguish it
# from the MLP gate, which is fused into gate_up_proj.
supported_lora_modules = [
"qkv_proj",
"o_proj",
"self_attn.gate_proj",
"gate_up_proj",
"down_proj",
"embed_tokens",
"lm_head",
]

def __init__(
self,
config: BeamConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
if get_beam_quantization(quant_config) is None:
quant_scheme = get_beam_quantization(quant_config)
if quant_scheme is None:
raise NotImplementedError(
"Beam supports BF16, compressed-tensors FP8 (per-channel "
"weights and dynamic per-token activations), or ModelOpt "
Expand All @@ -1313,8 +1331,14 @@ def __init__(
"Beam has not been validated with expert relocation or "
"redundant experts; use trivial expert placement"
)
if get_lora().enable_lora or get_lora().lora_paths:
raise NotImplementedError("LoRA adapters are not supported for Beam")
if (get_lora().enable_lora or get_lora().lora_paths) and (
quant_scheme == "nvfp4"
or get_lora().lora_backend not in ("csgmv", "triton")
):
raise NotImplementedError(
"Beam LoRA supports BF16 or compressed-tensors FP8 with "
"--lora-backend csgmv or triton and --moe-runner-backend triton"
)
self.config = config
self.model = BeamModel(config, quant_config, prefix=add_prefix("model", prefix))
self.start_layer = self.model.start_layer
Expand Down
1 change: 1 addition & 0 deletions python/sglang/srt/utils/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -4759,6 +4759,7 @@ def mxfp8_block_convert_required():
"wk",
"weights_proj",
"gate_proj",
"self_attn.gate_proj",
"up_proj",
"down_proj",
"qkv_proj",
Expand Down
Loading
Loading