Skip to content
Merged
13 changes: 10 additions & 3 deletions src/transformers/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
from .integrations.flash_paged import paged_attention_forward
from .integrations.flex_attention import flex_attention_forward
from .integrations.hub_kernels import allow_all_hub_kernels, is_kernel
from .integrations.moe import ALL_EXPERTS_FUNCTIONS
Comment thread
IlyasMoutawwakil marked this conversation as resolved.
Resolved
from .integrations.peft import maybe_load_adapters
from .integrations.sdpa_attention import sdpa_attention_forward
from .integrations.sdpa_paged import sdpa_attention_paged_forward
Expand All @@ -82,6 +83,7 @@
shard_and_distribute_module,
verify_tp_plan,
)
from .intergrations.finegrained_fp8 import ALL_FP8_EXPERTS_FUNCTIONS
Comment thread
IlyasMoutawwakil marked this conversation as resolved.
Resolved
from .loss.loss_utils import LOSS_MAPPING
from .modeling_flash_attention_utils import (
FLASH_ATTENTION_COMPATIBILITY_MATRIX,
Expand Down Expand Up @@ -1969,11 +1971,16 @@ def get_correct_attn_implementation(self, requested_attention: str | None, is_in

def get_correct_experts_implementation(self, requested_experts: str | None) -> str:
applicable_experts = "grouped_mm" if requested_experts is None else requested_experts
if applicable_experts not in ["eager", "grouped_mm", "batched_mm", "deepgemm"]:
base_experts_fns = list(
set(["eager"] + list(ALL_EXPERTS_FUNCTIONS.keys()) + list(ALL_FP8_EXPERTS_FUNCTIONS.keys()))
)
valid_experts_str_list = [f'`experts_implementation="{fn}"`' for fn in base_experts_fns]
valid_experts_str_list[-1] = "and " + valid_experts_str_list[-1]
valid_experts_str = ", ".join(valid_experts_str_list)
if applicable_experts not in base_experts_fns:
message = (
f'Specified `experts_implementation="{applicable_experts}"` is not supported. The only possible arguments are '
'`experts_implementation="eager"`, `"experts_implementation=grouped_mm"`, `"experts_implementation=batched_mm"` '
'and `"experts_implementation=deepgemm"`.'
f"{valid_experts_str}."
)
raise ValueError(message)

Expand Down
14 changes: 14 additions & 0 deletions tests/utils/test_modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2823,6 +2823,20 @@ def test_error_wrong_attn_implementation(self):

self.assertTrue('The only possible arguments are `attn_implementation="eager"' in str(cm.exception))

def test_registered_experts_implementation_is_valid(self):
from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS

def custom_experts_forward(*args, **kwargs):
pass

experts_implementation = "custom_experts"
model = BaseModel(PreTrainedConfig())

with patch.dict(ALL_EXPERTS_FUNCTIONS._global_mapping, {}, clear=False):
ALL_EXPERTS_FUNCTIONS.register(experts_implementation, custom_experts_forward)

self.assertEqual(model.get_correct_experts_implementation(experts_implementation), experts_implementation)

def test_not_available_flash(self):
if is_flash_attn_2_available():
self.skipTest(reason="Please uninstall flash-attn package to run test_not_available_flash")
Expand Down
Loading