Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/transformers/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4574,11 +4574,18 @@ def kernelize(self, mode=None):
def attach_hidden_kernels(module):
for name, fn in getattr(module, "_hidden_kernels", {}).items():
if name not in dict(module.named_children()):
module.register_module(name, fn)
if isinstance(fn, torch.nn.Module):
module._modules[name] = fn
else:
logger.warning_once(
f"Skipping kernel registration for '{name}': not an nn.Module. "
f"Consider decorating it with @use_kernel_func_from_hub."
)
Comment thread
jiqing-feng marked this conversation as resolved.
Resolved

def detach_hidden_kernels(module):
for name in getattr(module, "_hidden_kernels", {}):
delattr(module, name)
if name in module._modules:
del module._modules[name]

try:
self.apply(attach_hidden_kernels)
Expand Down
8 changes: 7 additions & 1 deletion src/transformers/models/gpt_oss/modeling_gpt_oss.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@
from ... import initialization as init
from ...cache_utils import Cache, DynamicCache
from ...generation import GenerationMixin
from ...integrations import use_experts_implementation, use_kernel_forward_from_hub, use_kernelized_func
from ...integrations import (
use_experts_implementation,
use_kernel_forward_from_hub,
use_kernel_func_from_hub,
use_kernelized_func,
)
from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask
from ...modeling_layers import (
GenericForSequenceClassification,
Expand Down Expand Up @@ -234,6 +239,7 @@ def _apply_rotary_emb(
return torch.cat((first_, second_), dim=-1)


@use_kernel_func_from_hub("rotary_pos_emb")
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
Expand Down
3 changes: 2 additions & 1 deletion src/transformers/models/gpt_oss/modular_gpt_oss.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from ... import initialization as init
from ...cache_utils import Cache, DynamicCache
from ...integrations import use_experts_implementation, use_kernel_forward_from_hub
from ...integrations import use_experts_implementation, use_kernel_forward_from_hub, use_kernel_func_from_hub
from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask
from ...modeling_outputs import (
MoeModelOutputWithPast,
Expand Down Expand Up @@ -171,6 +171,7 @@ def _apply_rotary_emb(
return torch.cat((first_, second_), dim=-1)


@use_kernel_func_from_hub("rotary_pos_emb")
Comment thread
jiqing-feng marked this conversation as resolved.
Resolved
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@
from torch.nn import functional as F

from ... import initialization as init
from ...integrations import use_experts_implementation, use_kernel_forward_from_hub, use_kernelized_func
from ...integrations import (
use_experts_implementation,
use_kernel_forward_from_hub,
use_kernel_func_from_hub,
use_kernelized_func,
)
from ...masking_utils import create_bidirectional_sliding_window_mask
from ...modeling_layers import GenericForTokenClassification, GradientCheckpointingLayer
from ...modeling_outputs import BaseModelOutput
Expand Down Expand Up @@ -137,6 +142,7 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)


@use_kernel_func_from_hub("rotary_pos_emb")
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
Expand Down
Loading