Skip to content

Commit e49740c

Browse files
wingliangithub-actions[bot]SunMarcIlyasMoutawwakil
authored andcommitted
Allow for registered experts from kernels hub (#45577)
* Allow for registered experts from kernels hub * remove deepgemm as that is also dynamic * Apply repo consistency fixes * Update src/transformers/modeling_utils.py * Update src/transformers/modeling_utils.py * Apply repo consistency fixes * Apply suggestion from @IlyasMoutawwakil * Apply repo consistency fixes * get rid of triton dependency * keep eager first --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Marc Sun <57196510+SunMarc@users.noreply.github.com> Co-authored-by: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com> Co-authored-by: IlyasMoutawwakil <moutawwakil.ilyas.tsi@gmail.com>
1 parent 90c938a commit e49740c

3 files changed

Lines changed: 33 additions & 10 deletions

File tree

‎src/transformers/integrations/finegrained_fp8.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
# limitations under the License.
1414
import torch
1515
import torch.nn as nn
16-
import triton
1716
from torch.nn import functional as F
1817

1918
from ..activations import ACT2FN
@@ -159,6 +158,11 @@ def _load_deepgemm_kernel():
159158
_deepgemm_available = True
160159

161160

161+
def _cdiv(a: int, b: int) -> int:
162+
"""Ceiling division."""
163+
return (a + b - 1) // b
164+
165+
162166
def w8a8_fp8_matmul(
163167
A: torch.Tensor,
164168
B: torch.Tensor,
@@ -603,26 +607,26 @@ def __init__(
603607
if self.has_gate:
604608
gu_proj_out, gu_proj_in = 2 * self.intermediate_dim, self.hidden_dim
605609
self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, gu_proj_out, gu_proj_in, dtype=dtype))
606-
gu_scale_out = triton.cdiv(gu_proj_out, self.block_size[0]) if self.block_size is not None else 1
607-
gu_scale_in = triton.cdiv(gu_proj_in, self.block_size[1]) if self.block_size is not None else 1
610+
gu_scale_out = _cdiv(gu_proj_out, self.block_size[0]) if self.block_size is not None else 1
611+
gu_scale_in = _cdiv(gu_proj_in, self.block_size[1]) if self.block_size is not None else 1
608612
self.gate_up_proj_scale_inv = nn.Parameter(
609613
torch.empty(self.num_experts, gu_scale_out, gu_scale_in, dtype=torch.float32)
610614
)
611615
self.register_parameter("gate_up_proj_bias", None)
612616
else:
613617
u_proj_out, u_proj_in = self.intermediate_dim, self.hidden_dim
614618
self.up_proj = nn.Parameter(torch.empty(self.num_experts, u_proj_out, u_proj_in, dtype=dtype))
615-
u_scale_out = triton.cdiv(u_proj_out, self.block_size[0]) if self.block_size is not None else 1
616-
u_scale_in = triton.cdiv(u_proj_in, self.block_size[1]) if self.block_size is not None else 1
619+
u_scale_out = _cdiv(u_proj_out, self.block_size[0]) if self.block_size is not None else 1
620+
u_scale_in = _cdiv(u_proj_in, self.block_size[1]) if self.block_size is not None else 1
617621
self.up_proj_scale_inv = nn.Parameter(
618622
torch.empty(self.num_experts, u_scale_out, u_scale_in, dtype=torch.float32)
619623
)
620624
self.register_parameter("up_proj_bias", None)
621625

622626
d_proj_out, d_proj_in = self.hidden_dim, self.intermediate_dim
623627
self.down_proj = nn.Parameter(torch.empty(self.num_experts, d_proj_out, d_proj_in, dtype=dtype))
624-
d_scale_out = triton.cdiv(d_proj_out, self.block_size[0]) if self.block_size is not None else 1
625-
d_scale_in = triton.cdiv(d_proj_in, self.block_size[1]) if self.block_size is not None else 1
628+
d_scale_out = _cdiv(d_proj_out, self.block_size[0]) if self.block_size is not None else 1
629+
d_scale_in = _cdiv(d_proj_in, self.block_size[1]) if self.block_size is not None else 1
626630
self.down_proj_scale_inv = nn.Parameter(
627631
torch.empty(self.num_experts, d_scale_out, d_scale_in, dtype=torch.float32)
628632
)

‎src/transformers/modeling_utils.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,12 @@
6666
)
6767
from .integrations.deepspeed import _load_state_dict_into_zero3_model
6868
from .integrations.eager_paged import eager_paged_attention_forward
69+
from .integrations.finegrained_fp8 import ALL_FP8_EXPERTS_FUNCTIONS
6970
from .integrations.flash_attention import flash_attention_forward
7071
from .integrations.flash_paged import paged_attention_forward
7172
from .integrations.flex_attention import flex_attention_forward
7273
from .integrations.hub_kernels import allow_all_hub_kernels, is_kernel
74+
from .integrations.moe import ALL_EXPERTS_FUNCTIONS
7375
from .integrations.peft import maybe_load_adapters
7476
from .integrations.sdpa_attention import sdpa_attention_forward
7577
from .integrations.sdpa_paged import sdpa_attention_paged_forward
@@ -1969,11 +1971,14 @@ def get_correct_attn_implementation(self, requested_attention: str | None, is_in
19691971

19701972
def get_correct_experts_implementation(self, requested_experts: str | None) -> str:
19711973
applicable_experts = "grouped_mm" if requested_experts is None else requested_experts
1972-
if applicable_experts not in ["eager", "grouped_mm", "batched_mm", "deepgemm"]:
1974+
base_experts_fns = ["eager"] + list(set(ALL_EXPERTS_FUNCTIONS.keys()) | set(ALL_FP8_EXPERTS_FUNCTIONS.keys()))
1975+
valid_experts_str_list = [f'`experts_implementation="{fn}"`' for fn in base_experts_fns]
1976+
valid_experts_str_list[-1] = "and " + valid_experts_str_list[-1]
1977+
valid_experts_str = ", ".join(valid_experts_str_list)
1978+
if applicable_experts not in base_experts_fns:
19731979
message = (
19741980
f'Specified `experts_implementation="{applicable_experts}"` is not supported. The only possible arguments are '
1975-
'`experts_implementation="eager"`, `"experts_implementation=grouped_mm"`, `"experts_implementation=batched_mm"` '
1976-
'and `"experts_implementation=deepgemm"`.'
1981+
f"{valid_experts_str}."
19771982
)
19781983
raise ValueError(message)
19791984

‎tests/utils/test_modeling_utils.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2823,6 +2823,20 @@ def test_error_wrong_attn_implementation(self):
28232823

28242824
self.assertTrue('The only possible arguments are `attn_implementation="eager"' in str(cm.exception))
28252825

2826+
def test_registered_experts_implementation_is_valid(self):
2827+
from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS
2828+
2829+
def custom_experts_forward(*args, **kwargs):
2830+
pass
2831+
2832+
experts_implementation = "custom_experts"
2833+
model = BaseModel(PreTrainedConfig())
2834+
2835+
with patch.dict(ALL_EXPERTS_FUNCTIONS._global_mapping, {}, clear=False):
2836+
ALL_EXPERTS_FUNCTIONS.register(experts_implementation, custom_experts_forward)
2837+
2838+
self.assertEqual(model.get_correct_experts_implementation(experts_implementation), experts_implementation)
2839+
28262840
def test_not_available_flash(self):
28272841
if is_flash_attn_2_available():
28282842
self.skipTest(reason="Please uninstall flash-attn package to run test_not_available_flash")

0 commit comments

Comments
 (0)