diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py index e7fee2633a9f..27b1c34207c9 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py @@ -421,27 +421,22 @@ def apply_weights( activation_type=activation_type, ) else: - quant_info = TritonMoeQuantInfo( - w13_weight=layer.w13_weight, - w2_weight=layer.w2_weight, - use_fp8_w8a8=True, - w13_scale=layer.w13_weight_scale, - w2_scale=layer.w2_weight_scale, - a13_scale=layer.w13_input_scale, - a2_scale=layer.w2_input_scale, - block_shape=self.weight_block_size, - ) + quant_info = self.get_triton_quant_info(layer) return self.runner.run(dispatch_output, quant_info) else: - quant_info = TritonMoeQuantInfo( - w13_weight=layer.w13_weight, - w2_weight=layer.w2_weight, - use_fp8_w8a8=True, - per_channel_quant=self.weight_quant.strategy - == QuantizationStrategy.CHANNEL, - w13_scale=layer.w13_weight_scale, - w2_scale=layer.w2_weight_scale, - a13_scale=layer.w13_input_scale, - a2_scale=layer.w2_input_scale, - ) - return self.runner.run(dispatch_output, quant_info) + return self.runner.run(dispatch_output, self.get_triton_quant_info(layer)) + + def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo: + # Shared with FusedMoEWithLoRA so both runners use the same base quantization. + return TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_fp8_w8a8=True, + per_channel_quant=self.weight_quant.strategy + == QuantizationStrategy.CHANNEL, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + block_shape=self.weight_block_size, + ) diff --git a/python/sglang/srt/lora/lora.py b/python/sglang/srt/lora/lora.py index 18a618941e62..90b7de35c1ac 100644 --- a/python/sglang/srt/lora/lora.py +++ b/python/sglang/srt/lora/lora.py @@ -141,33 +141,42 @@ def _is_non_gated_moe_weight(self, weight_name: str) -> bool: return self._moe_is_gated_by_layer.get(layer_id) is False def initialize_weights(self): + from sglang.srt.lora.utils import get_normalized_target_modules + model_path = self.config.path loader = DefaultModelLoader(self.load_config) revision = getattr(self.config.hf_config, "revision", None) # Get normalized target modules for filtering + normalized_target_modules = get_normalized_target_modules( + self.config.target_modules + ) for name, loaded_weight in loader._get_weights_iterator( DefaultModelLoader.Source( model_path, revision=revision, fall_back_to_pt=True ) ): - self._process_weight(name, loaded_weight) + self._process_weight(name, loaded_weight, normalized_target_modules) self._normalize_weights() def initialize_weights_from_tensors(self, tensors: Dict[str, torch.Tensor]): - for name, tensor in tensors.items(): - self._process_weight(name, tensor) - - self._normalize_weights() - - def _process_weight(self, name: str, loaded_weight: torch.Tensor): from sglang.srt.lora.utils import get_normalized_target_modules normalized_target_modules = get_normalized_target_modules( self.config.target_modules ) + for name, tensor in tensors.items(): + self._process_weight(name, tensor, normalized_target_modules) + + self._normalize_weights() + def _process_weight( + self, + name: str, + loaded_weight: torch.Tensor, + normalized_target_modules: set[str], + ): # Remap PEFT "unembed_tokens" key to "lm_head" so the weight is # recognized and loaded into the correct buffer. if "unembed_tokens" in name: @@ -462,7 +471,12 @@ def _normalize_in_proj_qkvz(self, weights: Dict[str, torch.Tensor]): def normalize_gate_up_proj( self, weight_names: List[str], weights: Dict[str, torch.Tensor] ): + from sglang.srt.lora.utils import PRESERVED_LORA_TARGET_NAMES + for weight_name in weight_names: + if any(f".{name}." in weight_name for name in PRESERVED_LORA_TARGET_NAMES): + # The attention output gate has no up_proj to stack with. + continue if "gate_proj" in weight_name: up_name = weight_name.replace("gate_proj", "up_proj") gate_up_name = weight_name.replace("gate_proj", "gate_up_proj") diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 6ef48343f612..02de87420bfa 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -347,6 +347,21 @@ def _has_shared_fused_moe(base_model: torch.nn.Module) -> bool: getattr(m, "is_shared_fused_moe", False) for m in base_model.modules() ) + @staticmethod + def _get_dense_module_layer_ids( + base_model: torch.nn.Module, module_name: str + ) -> Set[int]: + """Decoder layers containing this linear outside the fused experts.""" + from sglang.srt.layers.linear import LinearBase + + return { + layer_id + for name, module in base_model.named_modules() + if name.endswith(f".{module_name}") + and (layer_id := get_layer_id(name)) is not None + and isinstance(unwrap_lora_layer(module), LinearBase) + } + @staticmethod def _has_moe_module(base_model: torch.nn.Module) -> bool: # Config-only detection isn't reliable: some dense configs (e.g. @@ -610,10 +625,15 @@ def init_buffers(self, base_model: torch.nn.Module): # Cached once so the per-expert load path doesn't re-walk the HF # config for every adapter. self._num_experts_local: int = self._get_num_local_experts(base_model) + dense_layer_ids = { + name: self._get_dense_module_layer_ids(base_model, name) + for name in self.target_modules & {"gate_up_proj", "down_proj"} + } def alloc_per_layer( module_key: str, get_lora_shape_fn: Callable[[str, torch.nn.Module, int, int], Tuple[int]], + layer_ids: Optional[Set[int]] = None, ) -> List[Optional[torch.Tensor]]: return [ ( @@ -625,6 +645,7 @@ def alloc_per_layer( device=device, ) if idx in self.lora_layer_ids + and (layer_ids is None or idx in layer_ids) else None ) for idx in range(self.num_layer) @@ -650,10 +671,13 @@ def init_buffer( # Special handling for ambiguous target modules that can be in different contexts ambiguous_modules = {"gate_up_proj", "down_proj"} if module_name in ambiguous_modules and has_moe: - # Allocate shared expert version (3D) only when model has shared experts - if has_shared_experts: + # Hybrid models also need 3D buffers for their dense MLPs. + layer_ids = ( + None if has_shared_experts else dense_layer_ids[module_name] + ) + if layer_ids is None or layer_ids: buffer[module_name] = alloc_per_layer( - module_name, get_lora_shape_fn + module_name, get_lora_shape_fn, layer_ids ) # MoE expert version (4D) diff --git a/python/sglang/srt/lora/utils.py b/python/sglang/srt/lora/utils.py index f85c91da6072..edd9ea13ae0b 100644 --- a/python/sglang/srt/lora/utils.py +++ b/python/sglang/srt/lora/utils.py @@ -217,6 +217,8 @@ def get_default_hidden_dim( o_head_dim * config.num_attention_heads, config.hidden_size, ) + elif module_name == "self_attn.gate_proj": + return config.hidden_size, config.num_attention_heads * head_dim elif module_name == "gate_up_proj": inter = config.intermediate_size first_k = getattr(config, "first_k_dense_replace", None) @@ -300,6 +302,9 @@ def get_normalized_target_modules( Mapping a list of target module name to names of the normalized LoRA weights. Handles both base module names (e.g., "gate_proj") and prefixed module names (e.g., "feed_forward.gate_proj"). + The attention gate must be named "self_attn.gate_proj"; bare "gate_proj" + maps only to the MLP/MoE gate_up_proj. + Also handles PEFT shorthand strings like "all-linear" or "all" by returning {"all"} as a sentinel value. Callers that need a concrete module set should use :func:`auto_detect_lora_target_modules` to resolve the shorthand @@ -340,6 +345,10 @@ def get_normalized_target_modules( result = set() for name in target_modules: + qualified_name = ".".join(name.split(".")[-2:]) + if qualified_name in PRESERVED_LORA_TARGET_NAMES: + result.add(qualified_name) + continue base_name = name.split(".")[-1] normalized_name = params_mapping.get(base_name, base_name) result.add(normalized_name) @@ -401,6 +410,8 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s DSA_INDEXER_LORA_NAMES = frozenset( {"indexer.wq_b", "indexer.wk", "indexer.weights_proj"} ) +# Preserve the parent because the leaf alone names an MLP projection. +PRESERVED_LORA_TARGET_NAMES = frozenset({"self_attn.gate_proj"}) REPLICATED_LINEAR_LORA_NAMES = [ "fused_qkv_a_proj_with_mqa", "fc1_latent_proj", @@ -423,6 +434,7 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s "wo_ud", "in_proj", "in_proj_qkvz", + "self_attn.gate_proj", } ) @@ -449,6 +461,7 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s "kv_b_proj", } | DSA_INDEXER_LORA_NAMES + | PRESERVED_LORA_TARGET_NAMES ) @@ -483,7 +496,9 @@ def auto_detect_lora_target_modules(model: "torch.nn.Module") -> set: parts = name.split(".") leaf_name = parts[-1] parent_qualified_name = ".".join(parts[-2:]) - if parent_qualified_name in DSA_INDEXER_LORA_NAMES: + if parent_qualified_name in ( + DSA_INDEXER_LORA_NAMES | PRESERVED_LORA_TARGET_NAMES + ): raw_names.add(parent_qualified_name) elif leaf_name in dsa_indexer_leaf_names: # Bare DSA indexer leaf names are ambiguous across model diff --git a/python/sglang/srt/models/beam.py b/python/sglang/srt/models/beam.py index 91d2e2266fe7..7718f6276892 100644 --- a/python/sglang/srt/models/beam.py +++ b/python/sglang/srt/models/beam.py @@ -514,9 +514,12 @@ def __init__( self.sliding_window = config.sliding_window - 1 if self.apply_rope else -1 # MergedColumnParallelLinear shards each projection independently, but # does not replicate KV heads. Keep the existing quantized/replicated - # paths until their loading and backend contracts are validated. + # paths until their loading and backend contracts are validated. LoRA + # wraps qkv_proj and gate_proj separately. self.use_packed_projection = ( - quant_config is None and config.num_key_value_heads >= self.tp_size + quant_config is None + and config.num_key_value_heads >= self.tp_size + and not (get_lora().enable_lora or get_lora().lora_paths) ) if self.use_packed_projection: self.qkv_gate_proj = MergedColumnParallelLinear( @@ -944,9 +947,11 @@ def __init__( ) # Decoder layers whose outputs feed a speculative drafter (DFlash). self.layers_to_capture = () - # Deferred expert sums need the BF16 Triton runner's per-expert outputs. + # Deferred expert sums read base weights directly, bypassing LoRA. self.fuse_moe_all_reduce = ( - quant_config is None and config.num_experts_per_tok == 8 + quant_config is None + and config.num_experts_per_tok == 8 + and not (get_lora().enable_lora or get_lora().lora_paths) ) self._beam_all_reduce_comm = None # Offload wrappers load weights through each layer's regular forward. @@ -1286,6 +1291,18 @@ class BeamForCausalLM(nn.Module): "gate_up_proj": ["gate_proj", "up_proj"], } + # The attention output gate keeps its qualified name to distinguish it + # from the MLP gate, which is fused into gate_up_proj. + supported_lora_modules = [ + "qkv_proj", + "o_proj", + "self_attn.gate_proj", + "gate_up_proj", + "down_proj", + "embed_tokens", + "lm_head", + ] + def __init__( self, config: BeamConfig, @@ -1293,7 +1310,8 @@ def __init__( prefix: str = "", ): super().__init__() - if get_beam_quantization(quant_config) is None: + quant_scheme = get_beam_quantization(quant_config) + if quant_scheme is None: raise NotImplementedError( "Beam supports BF16, compressed-tensors FP8 (per-channel " "weights and dynamic per-token activations), or ModelOpt " @@ -1313,8 +1331,14 @@ def __init__( "Beam has not been validated with expert relocation or " "redundant experts; use trivial expert placement" ) - if get_lora().enable_lora or get_lora().lora_paths: - raise NotImplementedError("LoRA adapters are not supported for Beam") + if (get_lora().enable_lora or get_lora().lora_paths) and ( + quant_scheme == "nvfp4" + or get_lora().lora_backend not in ("csgmv", "triton") + ): + raise NotImplementedError( + "Beam LoRA supports BF16 or compressed-tensors FP8 with " + "--lora-backend csgmv or triton and --moe-runner-backend triton" + ) self.config = config self.model = BeamModel(config, quant_config, prefix=add_prefix("model", prefix)) self.start_layer = self.model.start_layer diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index db546dbd9b73..dd678a896d93 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4759,6 +4759,7 @@ def mxfp8_block_convert_required(): "wk", "weights_proj", "gate_proj", + "self_attn.gate_proj", "up_proj", "down_proj", "qkv_proj", diff --git a/test/registered/unit/lora/test_lora_manager_init_modules.py b/test/registered/unit/lora/test_lora_manager_init_modules.py index a1fe0ea0f066..2eb9197f0a7f 100644 --- a/test/registered/unit/lora/test_lora_manager_init_modules.py +++ b/test/registered/unit/lora/test_lora_manager_init_modules.py @@ -26,9 +26,13 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding from sglang.srt.lora import lora_manager as lora_manager_module +from sglang.srt.lora.lora import LoRAAdapter from sglang.srt.lora.lora_config import LoRAConfig from sglang.srt.lora.lora_manager import LoRAManager -from sglang.srt.lora.utils import auto_detect_lora_target_modules +from sglang.srt.lora.utils import ( + auto_detect_lora_target_modules, + get_normalized_target_modules, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -209,6 +213,7 @@ def test_auto_detect_lora_target_modules(self): attn = torch.nn.Module() attn.qkv_proj = _MockLinearBase() attn.o_proj = _MockLinearBase() + attn.gate_proj = _MockLinearBase() layer.self_attn = attn mlp = torch.nn.Module() mlp.gate_up_proj = _MockLinearBase() @@ -232,7 +237,57 @@ def test_auto_detect_lora_target_modules(self): detected = auto_detect_lora_target_modules(model) self.assertEqual( - detected, {"qkv_proj", "o_proj", "gate_up_proj", "down_proj", "lm_head"} + detected, + { + "qkv_proj", + "o_proj", + "self_attn.gate_proj", + "gate_up_proj", + "down_proj", + "lm_head", + }, + ) + + +class TestAttentionGateTargets(CustomTestCase): + def test_attention_gate_is_explicit_and_never_stacked_with_mlp(self): + """The attention gate used to become gate_up_proj, colliding with MLP LoRA.""" + self.assertEqual(get_normalized_target_modules(["gate_proj"]), {"gate_up_proj"}) + self.assertEqual( + get_normalized_target_modules( + ["model.layers.0.self_attn.gate_proj", "mlp.gate_proj"] + ), + {"self_attn.gate_proj", "gate_up_proj"}, + ) + prefix = "base_model.model.model.layers.0." + attn_key = prefix + "self_attn.gate_proj.lora_A.weight" + mlp_key = prefix + "mlp.gate_proj.lora_A.weight" + up_key = prefix + "mlp.up_proj.lora_A.weight" + attn, gate, up = [torch.full((2, 8), x) for x in (1.0, 2.0, 3.0)] + adapter = LoRAAdapter( + uid="gate", + config=LoRAConfig.from_dict( + dict( + peft_type="LORA", + r=2, + lora_alpha=2, + target_modules=["self_attn.gate_proj", "gate_proj", "up_proj"], + ) + ), + base_hf_config=SimpleNamespace(num_hidden_layers=1), + load_config=None, + lora_backend=None, + ) + adapter.initialize_weights_from_tensors( + {attn_key: attn, mlp_key: gate, up_key: up} + ) + weights = adapter.layers[0].weights + self.assertEqual( + set(weights), {attn_key, mlp_key.replace("gate_proj", "gate_up_proj")} + ) + torch.testing.assert_close(weights[attn_key], attn) + torch.testing.assert_close( + weights[mlp_key.replace("gate_proj", "gate_up_proj")], torch.cat((gate, up)) ) diff --git a/test/registered/unit/lora/test_lora_moe_inplace_unit.py b/test/registered/unit/lora/test_lora_moe_inplace_unit.py index e39e2d44dd40..d6781c9c4cf9 100644 --- a/test/registered/unit/lora/test_lora_moe_inplace_unit.py +++ b/test/registered/unit/lora/test_lora_moe_inplace_unit.py @@ -16,6 +16,7 @@ import torch from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.test.test_utils import CustomTestCase def _make_base_layer() -> types.SimpleNamespace: @@ -48,7 +49,7 @@ def _make_base_layer() -> types.SimpleNamespace: ) -class FusedMoEWithLoRAInplaceTest(unittest.TestCase): +class FusedMoEWithLoRAInplaceTest(CustomTestCase): def test_constructor_forces_inplace_off(self): from sglang.srt.lora.layers import FusedMoEWithLoRA @@ -59,6 +60,43 @@ def test_constructor_forces_inplace_off(self): self.assertIs(layer.moe_runner_config, base_layer.moe_runner_config) self.assertFalse(layer.moe_runner_config.inplace) + def test_compressed_fp8_initializes_with_real_quant_info(self): + # The quant method delegates to the scheme, which previously lacked the getter. + from compressed_tensors.quantization import QuantizationArgs + + from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsFusedMoEMethod, + ) + from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8_fp8_moe import ( + CompressedTensorsW8A8Fp8MoE, + ) + from sglang.srt.lora.layers import FusedMoEWithLoRA + + base_layer = _make_base_layer() + base_layer.quant_method = CompressedTensorsFusedMoEMethod(None) + base_layer.scheme = CompressedTensorsW8A8Fp8MoE( + weight_quant=QuantizationArgs( + type="float", strategy="channel", dynamic=False + ), + input_quant=QuantizationArgs(type="float", strategy="token", dynamic=True), + ) + base_layer.w13_weight = torch.zeros(8, 64, 64, dtype=torch.float8_e4m3fn) + base_layer.w2_weight = torch.zeros(8, 64, 32, dtype=torch.float8_e4m3fn) + base_layer.w13_weight_scale = torch.ones(8, 64, 1) + base_layer.w2_weight_scale = torch.ones(8, 64, 1) + base_layer.w13_input_scale = base_layer.w2_input_scale = None + + layer = FusedMoEWithLoRA(base_layer, types.SimpleNamespace(name="triton")) + + self.assertTrue(layer._quant_info.use_fp8_w8a8) + self.assertTrue(layer._quant_info.per_channel_quant) + self.assertIsNone(layer._quant_info.block_shape) + self.assertIs(layer._quant_info.w13_weight, base_layer.w13_weight) + self.assertIs(layer._quant_info.w2_weight, base_layer.w2_weight) + self.assertIs(layer._quant_info.w13_scale, base_layer.w13_weight_scale) + self.assertIs(layer._quant_info.w2_scale, base_layer.w2_weight_scale) + self.assertFalse(layer.moe_runner_config.inplace) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/lora/test_lora_pp.py b/test/registered/unit/lora/test_lora_pp.py index 7ee6e4178aae..e8414354f1f6 100644 --- a/test/registered/unit/lora/test_lora_pp.py +++ b/test/registered/unit/lora/test_lora_pp.py @@ -13,6 +13,7 @@ # ============================================================================== import unittest +from copy import deepcopy from types import SimpleNamespace from unittest import mock @@ -23,6 +24,8 @@ maybe_stub_sgl_kernel() +from sglang.srt.layers.linear import ReplicatedLinear +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.lora.eviction_policy import get_eviction_policy from sglang.srt.lora.lora_drainer import LoRADrainer from sglang.srt.lora.lora_manager import LoRAManager @@ -284,6 +287,59 @@ def _load_into_mem_pool(pool, uid, adapter): class TestStagePool(CustomTestCase): + def test_hybrid_mlp_allocates_dense_and_expert_buffers_on_local_stage(self): + """A dense prefix needs 3D buffers even without shared experts; off-stage + layers must still be left unallocated.""" + config = SimpleNamespace( + num_hidden_layers=3, + hidden_size=8, + num_attention_heads=2, + num_key_value_heads=2, + intermediate_size=16, + moe_intermediate_size=8, + num_experts=2, + ) + model = torch.nn.Module() + model.config = config + model.register_parameter("anchor", torch.nn.Parameter(torch.empty(0))) + dense = torch.nn.Module() + dense.gate_up_proj = ReplicatedLinear(8, 32, bias=False) + dense.down_proj = ReplicatedLinear(16, 8, bias=False) + experts = FusedMoE.__new__(FusedMoE) + torch.nn.Module.__init__(experts) + model.layers = torch.nn.ModuleList([dense, experts, deepcopy(dense)]) + for has_dense in (True, False): + if not has_dense: + model.layers[0] = torch.nn.Module() + model.layers[2] = torch.nn.Module() + with ( + self.subTest(has_dense=has_dense), + get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1), + ): + pool = LoRAMemoryPool( + base_hf_config=config, + max_loras_per_batch=1, + dtype=torch.float32, + max_lora_rank=2, + target_modules={"gate_up_proj", "down_proj"}, + base_model=model, + eviction_policy="lru", + lora_added_tokens_size=0, + lora_layer_ids=[0, 1], + has_embed_tokens_lora=False, + has_lm_head_lora=False, + ) + for buffer in (pool.A_buffer, pool.B_buffer): + for name in ("gate_up_proj", "down_proj"): + self.assertEqual(name in buffer, has_dense) + if has_dense: + self.assertEqual(buffer[name][0].ndim, 3) + self.assertIsNone(buffer[name][1]) + self.assertIsNone(buffer[name][2]) + self.assertEqual(buffer[name + "_moe"][0].ndim, 4) + self.assertEqual(buffer[name + "_moe"][1].ndim, 4) + self.assertIsNone(buffer[name + "_moe"][2]) + def test_each_stage_allocates_and_loads_only_its_part(self): first = _stage_pool(layers=[0, 1], embed_tokens=True) last = _stage_pool(layers=[2, 3], lm_head=True, added_tokens_size=1) diff --git a/test/registered/unit/lora/test_lora_startup_contracts.py b/test/registered/unit/lora/test_lora_startup_contracts.py index 3386aecb7994..7e31540b636b 100644 --- a/test/registered/unit/lora/test_lora_startup_contracts.py +++ b/test/registered/unit/lora/test_lora_startup_contracts.py @@ -13,11 +13,13 @@ import torch +from sglang.srt.configs.beam import BeamConfig from sglang.srt.configs.laguna import LagunaConfig from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.lora.backend.uno_cublas_backend import UnoCublasLoRABackend from sglang.srt.lora.eviction_policy import get_eviction_policy from sglang.srt.lora.lora_manager import LoRAManager +from sglang.srt.models.beam import BeamModel from sglang.srt.models.laguna import LagunaModel from sglang.srt.runtime_context import get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci @@ -132,5 +134,69 @@ def test_attention_projections_follow_each_layers_head_count(self): ) +class TestBeamLoraConstruction(CustomTestCase): + def test_lora_keeps_wrappable_projections_and_expert_dispatch(self): + """Packed projections hide adapter targets; deferred sums bypass LoRA. + + Tiny serving fixtures never enter the hidden-size-5120 fusion path. + """ + config = BeamConfig( + vocab_size=128, + hidden_size=5120, + num_hidden_layers=1, + num_attention_heads=8, + num_key_value_heads=2, + head_dim=64, + layer_types=["full_attention"], + mlp_layer_types=["sparse"], + ) + pp_group = SimpleNamespace( + is_first_rank=True, is_last_rank=True, rank_in_group=0, world_size=1 + ) + # CUDA metadata suffices for eligibility; no GPU storage or collective. + hidden = SimpleNamespace( + is_cuda=True, + dtype=torch.bfloat16, + shape=(4, 5120), + device=torch.device("cuda"), + is_contiguous=lambda: True, + data_ptr=lambda: 16, + ) + for enabled in (False, True): + with ( + self.subTest(enable_lora=enabled), + get_context().override_server_args( + enable_lora=enabled, disable_custom_all_reduce=False + ), + get_parallel().override( + tp_size=2, + tp_rank=0, + attn_tp_size=2, + attn_tp_rank=0, + attn_cp_size=1, + attn_dp_size=1, + moe_tp_size=2, + moe_ep_size=1, + pp_group=pp_group, + ), + patch("torch.cuda.is_available", return_value=False), + patch("torch.cuda.get_device_capability", return_value=(9, 0)), + patch( + "sglang.srt.models.beam.BeamMoE", return_value=torch.nn.Identity() + ), + torch.device("meta"), + ): + model = BeamModel(config) + projections = set(dict(model.layers[0].self_attn.named_children())) + self.assertEqual( + projections & {"qkv_gate_proj", "qkv_proj", "gate_proj"}, + {"qkv_proj", "gate_proj"} if enabled else {"qkv_gate_proj"}, + ) + attention_limit, expert_limit = model._all_reduce_limits(hidden) + self.assertGreater(attention_limit, 0) + self.assertEqual(expert_limit > 0, not enabled) + self.assertTrue(model._can_replace_mlp_all_reduce()) + + if __name__ == "__main__": unittest.main()