Repository navigation
Transformers 4.54 and 4.55 and gpt-oss #52
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 34 commits
54c815f
2fbc83e
c2de155
075247d
085d871
df15230
f06a169
6ccc95f
5ac0170
fd2a30d
983ce8d
389be21
c7b64e0
e6281c5
9abe096
b68ff86
e89b44f
3a1f0ac
8a0239e
d674231
cedb778
07ac109
5080be8
52ac0e0
747d7cd
f378ca7
5ac17c0
0a5ffba
23b0700
acfda06
b6f80bf
aa619c3
4bc8bd9
e791f8b
2f8ad72
f06d999
c1f0c14
3775b6e
6788154
96040e7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| import warnings | ||
| from collections import defaultdict | ||
| from functools import wraps | ||
|
|
||
| from transformers.utils.generic import _CAN_RECORD_REGISTRY, OutputRecorder, logger | ||
|
|
||
|
|
||
| def check_model_inputs_patched(func): | ||
| @wraps(func) | ||
| def wrapper(self, *args, **kwargs): | ||
| use_cache = ( | ||
| kwargs["use_cache"] if kwargs.get("use_cache") is not None else getattr(self.config, "use_cache", None) | ||
| ) | ||
| if use_cache is not None: | ||
| if getattr(self, "gradient_checkpointing", False) and self.training and use_cache: | ||
| logger.warning_once( | ||
| "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." | ||
| ) | ||
| use_cache = False | ||
|
|
||
| # Prevent passing use_cache twice | ||
| if "use_cache" in func.__code__.co_varnames: | ||
| use_cache_idx = func.__code__.co_varnames.index("use_cache") - 1 # minus 1 for 'self' | ||
| if len(args) > use_cache_idx: | ||
| args = list(args) | ||
| args[use_cache_idx] = use_cache | ||
| args = tuple(args) | ||
| else: | ||
| kwargs["use_cache"] = use_cache | ||
|
|
||
| return_dict = kwargs.pop("return_dict", None) | ||
| if return_dict is None: | ||
| return_dict = getattr(self.config, "return_dict", True) | ||
|
|
||
| all_args = kwargs.copy() | ||
| if "kwargs" in all_args: | ||
| for k, v in all_args["kwargs"].items(): | ||
| all_args[k] = v | ||
|
|
||
| capture_flags = _CAN_RECORD_REGISTRY.get(str(self.__class__), {}) # there is a weak ref for executorch | ||
| if capture_flags is None: | ||
| capture_flags = {} | ||
|
IlyasMoutawwakil marked this conversation as resolved.
Resolved
|
||
|
|
||
| recordable_keys = { | ||
| f"output_{k}": all_args.get( | ||
| f"output_{k}", | ||
| getattr( | ||
| self.config, | ||
| f"output_{k}", | ||
| all_args.get("output_attentions", getattr(self.config, "output_attentions", False)), | ||
| ), | ||
| ) | ||
| for k in capture_flags | ||
| } | ||
|
|
||
| # We let cross attentions to be saved separately because some models add `cross-attn` layer | ||
| # when certain condtions are met. Let's output cross attention if attentions are requested (for BC) | ||
| if "output_attentions" in recordable_keys: | ||
| recordable_keys["output_cross_attentions"] = recordable_keys["output_attentions"] | ||
|
|
||
| collected_outputs = defaultdict(tuple) | ||
| monkey_patched_layers = [] | ||
|
|
||
| # Check attention implementation is properly set for capturing attention outputs | ||
| if recordable_keys.get("output_attentions", False): | ||
| supported_attn = ["eager", "eager_paged", "flex_attention"] | ||
| config_attn = getattr(self.config, "_attn_implementation", None) | ||
| sub_configs = [getattr(self.config, key, None) for key in self.config.sub_configs] | ||
| sub_configs_attn = [ | ||
| getattr(config, "_attn_implementation", None) for config in sub_configs if config is not None | ||
| ] | ||
| if config_attn not in supported_attn or any(attn not in supported_attn for attn in sub_configs_attn): | ||
| warnings.warn( | ||
| f"`output_attentions=True` is not supported with `attn_implementation` other than {supported_attn}. " | ||
| "Please use `model.set_attn_implementation('eager')` to enable capturing attention outputs.", | ||
| UserWarning, | ||
| stacklevel=2, | ||
| ) | ||
|
|
||
| def make_capture_wrapper(module, orig_forward, key, index): | ||
| @wraps(orig_forward) | ||
| def wrapped_forward(*args, **kwargs): | ||
| if key == "hidden_states" and len(collected_outputs[key]) == 0: | ||
| collected_outputs[key] += (args[0],) | ||
| output = orig_forward(*args, **kwargs) | ||
| if not isinstance(output, tuple): | ||
| collected_outputs[key] += (output,) | ||
| elif output[index] is not None: | ||
| if key not in collected_outputs: | ||
| collected_outputs[key] = (output[index],) | ||
| else: | ||
| collected_outputs[key] += (output[index],) | ||
| return output | ||
|
|
||
| return wrapped_forward | ||
|
|
||
| if any(recordable_keys.values()): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not sure to understand why we need all of this, if you could add some comment to explain that would be awesome 🙏
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I just patched/fixed the function to accept use_cache as a positional argument and not fail with some models 😭 |
||
| capture_tasks = [] | ||
| for key, layer_specs in capture_flags.items(): | ||
| if not recordable_keys.get(f"output_{key}", False): | ||
| continue | ||
| if not isinstance(layer_specs, list): | ||
| layer_specs = [layer_specs] | ||
| for specs in layer_specs: | ||
| if not isinstance(specs, OutputRecorder): | ||
| index = 0 if "hidden_states" in key else 1 | ||
| class_name = None if not isinstance(specs, str) else specs | ||
| target_class = specs if not isinstance(specs, str) else None | ||
| specs = OutputRecorder(target_class=target_class, index=index, class_name=class_name) | ||
| capture_tasks.append((key, specs)) | ||
|
|
||
| for name, module in self.named_modules(): | ||
| for key, specs in capture_tasks: | ||
| # The second check is for multimodals where only backbone layer suffix is available | ||
| if (specs.target_class is not None and isinstance(module, specs.target_class)) or ( | ||
| specs.class_name is not None and name.endswith(specs.class_name) | ||
| ): | ||
| if specs.layer_name is not None and specs.layer_name not in name: | ||
| continue | ||
| # Monkey patch forward | ||
| original_forward = module.forward | ||
| module.forward = make_capture_wrapper(module, original_forward, key, specs.index) | ||
| monkey_patched_layers.append((module, original_forward)) | ||
|
|
||
| outputs = func(self, *args, **kwargs) | ||
| # Restore original forward methods | ||
| for module, original_forward in monkey_patched_layers: | ||
| module.forward = original_forward | ||
|
|
||
| # Inject collected outputs into model output | ||
| for key in collected_outputs: | ||
| if key == "hidden_states": | ||
| if hasattr(outputs, "vision_hidden_states"): | ||
| collected_outputs[key] = collected_outputs[key][:-1] | ||
| collected_outputs[key] += (outputs.vision_hidden_states,) | ||
| elif hasattr(outputs, "last_hidden_state"): | ||
| collected_outputs[key] = collected_outputs[key][:-1] | ||
| collected_outputs[key] += (outputs.last_hidden_state,) | ||
|
|
||
| outputs[key] = collected_outputs[key] | ||
| elif key == "attentions": | ||
| if isinstance(capture_flags[key], list) and len(capture_flags[key]) == 2: | ||
| outputs[key] = collected_outputs[key][0::2] | ||
| outputs["cross_" + key] = collected_outputs[key][1::2] | ||
| else: | ||
| outputs[key] = collected_outputs[key] | ||
| else: | ||
| outputs[key] = collected_outputs[key] | ||
| if return_dict is False: | ||
| outputs = outputs.to_tuple() | ||
| return outputs | ||
|
|
||
| return wrapper | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| from __future__ import annotations | ||
|
IlyasMoutawwakil marked this conversation as resolved.
Resolved
|
||
|
|
||
| from optimum.utils import DummyPastKeyValuesGenerator, NormalizedTextConfig, is_transformers_version | ||
|
|
||
|
|
||
| class GPTBigCodeDummyPastKeyValuesGenerator(DummyPastKeyValuesGenerator): | ||
| def __init__(self, task: str, normalized_config: NormalizedTextConfig, **kwargs): | ||
| super().__init__(task=task, normalized_config=normalized_config, **kwargs) | ||
| self.multi_query = normalized_config.multi_query | ||
|
|
||
| def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): | ||
| if is_transformers_version("<", "4.54"): | ||
| if self.multi_query: | ||
| shape = ( | ||
| self.batch_size, | ||
| self.sequence_length, | ||
| self.hidden_size // self.num_attention_heads * 2, | ||
| ) | ||
| else: | ||
| shape = ( | ||
| self.batch_size, | ||
| self.num_attention_heads, | ||
| self.sequence_length, | ||
| self.hidden_size // self.num_attention_heads * 2, | ||
| ) | ||
| pkv = [ | ||
| self.random_float_tensor(shape, framework=framework, dtype=float_dtype) for _ in range(self.num_layers) | ||
| ] | ||
|
|
||
| else: | ||
| if self.multi_query: | ||
| shape = ( | ||
| self.batch_size, | ||
| 1, | ||
| self.sequence_length, | ||
| self.hidden_size // self.num_attention_heads, | ||
| ) | ||
| else: | ||
| shape = ( | ||
| self.batch_size, | ||
| self.num_attention_heads, | ||
| self.sequence_length, | ||
| self.hidden_size // self.num_attention_heads, | ||
| ) | ||
| pkv = [ | ||
| ( | ||
| self.random_float_tensor(shape, framework=framework, dtype=float_dtype), | ||
| self.random_float_tensor(shape, framework=framework, dtype=float_dtype), | ||
| ) | ||
| for _ in range(self.num_layers) | ||
| ] | ||
|
|
||
| return pkv | ||
Uh oh!
There was an error while loading. Please reload this page.