Skip to content
6 changes: 5 additions & 1 deletion src/transformers/configuration_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1116,11 +1116,15 @@ def register_for_auto_class(cls, auto_class="AutoConfig"):

def _get_generation_parameters(self) -> dict[str, Any]:
"""
Gets the non-default generation parameters on the PreTrainedConfig instance
Checks if there are generation parameters in `PreTrainedConfig` instance. Note that
we should not save generation params in PreTrainedConfig, and we will raise error
if there are any.
"""
generation_params = {}
default_config = self.__class__().to_dict() if not self.has_no_defaults_at_init else {}
for key in GenerationConfig._get_default_generation_params().keys():
if key == "use_cache":
continue # common key for most models
Comment on lines +1126 to +1127

@zucchini-nlp zucchini-nlp Dec 23, 2025 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

btw, could we remove use_cache as a key from config and instead keep it only in generation_config? Imo it isn't really a model attribute

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm unsure. I understand the point but that would be hugely breaking for a lot of users and external libraries - on the other hand we could include this for v5 if you feel motivated 🤔

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it will definitely break stuff 😿 Just a thought for now

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so, this is a workaround we have to not throw errors that model config contains generation params. We won't throw errors anyway if use_cache is in model config signature but unfortunately many don't have it documented

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha, maybe at some point we can 👀

if hasattr(self, key) and getattr(self, key) is not None and key not in default_config:
generation_params[key] = getattr(self, key)

Expand Down
80 changes: 46 additions & 34 deletions src/transformers/generation/configuration_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ class GenerationConfig(PushToHubMixin):

</Tip>

Note: the configuration field that are still `None` will be overriden by `GenerationConfig._get_default_generation_params()`
during the generation loop. If you want to use different values for these fields, make sure to explicitly set them in the
generation config.

Arg:
> Parameters that control the length of the output

Expand Down Expand Up @@ -129,14 +133,14 @@ class GenerationConfig(PushToHubMixin):

> Parameters that control the generation strategy used

do_sample (`bool`, defaults to `False`):
do_sample (`bool`):
Whether or not to use sampling ; use greedy decoding otherwise.
num_beams (`int`, *optional*):
Number of beams for beam search. 1 means no beam search.

> Parameters that control the cache

use_cache (`bool`, defaults to `True`):
use_cache (`bool`):
Whether or not the model should use the past last key/values attentions (if applicable to the model) to
speed up decoding.
cache_implementation (`str`, *optional*):
Expand Down Expand Up @@ -206,7 +210,7 @@ class GenerationConfig(PushToHubMixin):
bad_words_ids (`list[list[int]]`, *optional*):
List of list of token ids that are not allowed to be generated. Check
[`~generation.NoBadWordsLogitsProcessor`] for further documentation and examples.
renormalize_logits (`bool`, defaults to `False`):
renormalize_logits (`bool`):
Whether to renormalize the logits after applying all the logits processors (including the custom
ones). It's highly recommended to set this flag to `True` as the search algorithms suppose the score logits
are normalized but some logit processors break the normalization.
Expand All @@ -217,7 +221,7 @@ class GenerationConfig(PushToHubMixin):
forced_eos_token_id (`int` or list[int]`, *optional*, defaults to `model.config.forced_eos_token_id`):
The id of the token to force as the last generated token when `max_length` is reached. Optionally, use a
list to set multiple *end-of-sequence* tokens.
remove_invalid_values (`bool`, defaults to `model.config.remove_invalid_values`):
remove_invalid_values (`bool`):
Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash.
Note that using `remove_invalid_values` can slow down generation.
exponential_decay_length_penalty (`tuple(int, float)`, *optional*):
Expand All @@ -234,7 +238,7 @@ class GenerationConfig(PushToHubMixin):
Dictionary that maps a sequence of tokens to its bias term. Positive biases increase the odds of the
sequence being selected, while negative biases do the opposite. Check
[`~generation.SequenceBiasLogitsProcessor`] for further documentation and examples.
token_healing (`bool`, defaults to `False`):
token_healing (`bool`):
Heal tail tokens of prompts by replacing them with their appropriate extensions.
This enhances the quality of completions for prompts affected by greedy tokenization bias.
guidance_scale (`float`, *optional*):
Expand All @@ -248,20 +252,20 @@ class GenerationConfig(PushToHubMixin):

> Parameters that define the output variables of generate

num_return_sequences (`int`, *optional*, defaults to 1):
num_return_sequences (`int`, *optional*):
The number of independently computed returned sequences for each element in the batch.
output_attentions (`bool`, defaults to `False`):
output_attentions (`bool`):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more details.
output_hidden_states (`bool`, defaults to `False`):
output_hidden_states (`bool`):
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more details.
output_scores (`bool`, defaults to `False`):
output_scores (`bool`):
Whether or not to return the prediction scores. See `scores` under returned tensors for more details.
output_logits (`bool`, defaults to `False`):
output_logits (`bool`):
Whether or not to return the unprocessed prediction logit scores. See `logits` under returned tensors for
more details.
return_dict_in_generate (`bool`, defaults to `False`):
return_dict_in_generate (`bool`):
Whether or not to return a [`~utils.ModelOutput`], as opposed to returning exclusively the generated
sequence. This flag must be set to `True` to return the generation cache (when `use_cache` is `True`)
or optional outputs (see flags starting with `output_`)
Expand All @@ -286,7 +290,7 @@ class GenerationConfig(PushToHubMixin):
(e.g. multilingual models with different target languages in one batch)

> Generation parameters exclusive to assistant generation
is_assistant (`bool`, defaults to `False`):
is_assistant (`bool`):
Whether the model is an assistant (draft) model.
num_assistant_tokens (`int`, *optional*):
Defines the number of _speculative tokens_ that shall be generated by the assistant model before being
Expand Down Expand Up @@ -327,7 +331,7 @@ class GenerationConfig(PushToHubMixin):
compile_config (CompileConfig, *optional*):
If using a compilable cache, this controls how `generate` will `compile` the forward pass for faster
inference.
disable_compile (`bool`, defaults to `False`):
disable_compile (`bool`):
Whether to disable the automatic compilation of the forward pass. Automatic compilation happens when
specific criteria are met, including using a compilable cache. Please open an issue if you find the
need to use this flag.
Expand All @@ -346,11 +350,11 @@ def __init__(self, **kwargs):
self.stop_strings = kwargs.pop("stop_strings", None)

# Parameters that control the generation strategy used
self.do_sample = kwargs.pop("do_sample", False)
self.do_sample = kwargs.pop("do_sample", None)
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved
self.num_beams = kwargs.pop("num_beams", None)

# Parameters that control the cache
self.use_cache = kwargs.pop("use_cache", True)
self.use_cache = kwargs.pop("use_cache", None)
self.cache_implementation = kwargs.pop("cache_implementation", None)
self.cache_config = kwargs.pop("cache_config", None)

Expand All @@ -368,28 +372,28 @@ def __init__(self, **kwargs):
self.length_penalty = kwargs.pop("length_penalty", None)
self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", None)
self.bad_words_ids = kwargs.pop("bad_words_ids", None)
self.renormalize_logits = kwargs.pop("renormalize_logits", False)
self.renormalize_logits = kwargs.pop("renormalize_logits", None)
self.forced_bos_token_id = kwargs.pop("forced_bos_token_id", None)
self.forced_eos_token_id = kwargs.pop("forced_eos_token_id", None)
self.remove_invalid_values = kwargs.pop("remove_invalid_values", False)
self.remove_invalid_values = kwargs.pop("remove_invalid_values", None)
self.exponential_decay_length_penalty = kwargs.pop("exponential_decay_length_penalty", None)
self.suppress_tokens = kwargs.pop("suppress_tokens", None)
self.begin_suppress_tokens = kwargs.pop("begin_suppress_tokens", None)
self.sequence_bias = kwargs.pop("sequence_bias", None)
self.token_healing = kwargs.pop("token_healing", False)
self.token_healing = kwargs.pop("token_healing", None)
self.guidance_scale = kwargs.pop("guidance_scale", None)

self.watermarking_config = kwargs.pop("watermarking_config", None)
if isinstance(self.watermarking_config, dict):
self.watermarking_config = WatermarkingConfig.from_dict(self.watermarking_config)

# Parameters that define the output variables of `generate`
self.num_return_sequences = kwargs.pop("num_return_sequences", 1)
self.output_attentions = kwargs.pop("output_attentions", False)
self.output_hidden_states = kwargs.pop("output_hidden_states", False)
self.output_scores = kwargs.pop("output_scores", False)
self.output_logits = kwargs.pop("output_logits", False)
self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", False)
self.num_return_sequences = kwargs.pop("num_return_sequences", None)
self.output_attentions = kwargs.pop("output_attentions", None)
self.output_hidden_states = kwargs.pop("output_hidden_states", None)
self.output_scores = kwargs.pop("output_scores", None)
self.output_logits = kwargs.pop("output_logits", None)
self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", None)
Comment on lines -387 to +396

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I'm a bit lost into the motivation the PR and making those values to None (which is usually not a super good idea if they are boolean by essence) - could you give me a bit of context on the PR?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The motivation was to fix a ton of edge cases when creating a generation config. In the past we had ugly workaround to allow users to change a generation param back to default if different value was saved on the hub (for ex #42762).

That was fixed in #42702 and I deleted a lot of code, and defined a clear priority list, Next step will be to disallow users to pass both: a generation config and kwargs. Instead ask to put all kwargs in config. The below is accepted rn and is a bad practice imo

conf = GenerationConfig(do_sample=False)
text = model.generate(input_ids, generation_config=conf, max_new_tokens=128, temperature=1.6)


# Special tokens that can be used at generation time
self.pad_token_id = kwargs.pop("pad_token_id", None)
Expand All @@ -401,7 +405,7 @@ def __init__(self, **kwargs):
self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None)

# Assistant generation
self.is_assistant = kwargs.pop("is_assistant", False)
self.is_assistant = kwargs.pop("is_assistant", None)
self.num_assistant_tokens = kwargs.pop("num_assistant_tokens", None)
self.num_assistant_tokens_schedule = kwargs.pop("num_assistant_tokens_schedule", None)
self.assistant_confidence_threshold = kwargs.pop("assistant_confidence_threshold", None)
Expand All @@ -413,7 +417,7 @@ def __init__(self, **kwargs):

# Performance
self.compile_config = kwargs.pop("compile_config", None)
self.disable_compile = kwargs.pop("disable_compile", False)
self.disable_compile = kwargs.pop("disable_compile", None)

# Deprecated (moved to the Hub). TODO remove for v5
self.low_memory = kwargs.pop("low_memory", None)
Expand Down Expand Up @@ -483,7 +487,7 @@ def get_generation_mode(self, assistant_model: Optional["PreTrainedModel"] = Non
if self.constraints is not None or self.force_words_ids is not None:
generation_mode = GenerationMode.CONSTRAINED_BEAM_SEARCH
elif self.num_beams is None or self.num_beams == 1:
if not self.do_sample:
if self.do_sample is not True:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if self.do_sample is not True:
if self.do_sample is False:

super nit, seen this multiple times

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, it should be if not self.do_sample haha

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if not do_sample will result in false positives when do_sample is None 😢 Gotta be explicitly checking for not True, since the value of None is same as having False, i.e. the new default value

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a small comment then, that's such a weird case lol.

if (
self.top_k is not None
and self.top_k > 1
Expand All @@ -498,7 +502,7 @@ def get_generation_mode(self, assistant_model: Optional["PreTrainedModel"] = Non
else:
if self.num_beam_groups is not None and self.num_beam_groups > 1:
generation_mode = GenerationMode.GROUP_BEAM_SEARCH
elif self.do_sample:
elif self.do_sample is True:
generation_mode = GenerationMode.BEAM_SAMPLE
else:
generation_mode = GenerationMode.BEAM_SEARCH
Expand Down Expand Up @@ -537,6 +541,7 @@ def _get_default_generation_params() -> dict[str, Any]:
"max_length": 20,
"min_length": 0,
"do_sample": False,
"use_cache": True,
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved
"early_stopping": False,
"num_beams": 1,
"temperature": 1.0,
Expand Down Expand Up @@ -616,7 +621,10 @@ def validate(self, strict=False):

# 2. Validation of attribute combinations
# 2.1. detect sampling-only parameterization when not in sampling mode
if not self.do_sample:

# Note that we check `is not True` in purpose. Boolean fields can also be `None` so we
# have to be explicit. Value of `None` is same as having `False`, i.e. the default value
if self.do_sample is not True:
greedy_wrong_parameter_msg = (
"`do_sample` is set not to set `True`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only "
"used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`."
Expand Down Expand Up @@ -662,21 +670,25 @@ def validate(self, strict=False):
)

# 2.4. check `num_return_sequences`
if self.num_return_sequences > 1:
if self.num_return_sequences is not None and self.num_return_sequences > 1:
if self.num_beams is None or self.num_beams == 1:
if not self.do_sample:
raise ValueError(
"Greedy methods (do_sample != True) without beam search do not support "
f"`num_return_sequences` different than 1 (got {self.num_return_sequences})."
)
elif self.num_beams is not None and self.num_return_sequences > self.num_beams:
elif (
self.num_beams is not None
and self.num_return_sequences is not None
and self.num_return_sequences > self.num_beams
):
raise ValueError(
f"`num_return_sequences` ({self.num_return_sequences}) has to be smaller or equal to `num_beams` "
f"({self.num_beams})."
)

# 2.5. check cache-related arguments
if not self.use_cache:
if self.use_cache is not True:
# In this case, all cache-related arguments should be unset. However, since `use_cache=False` is often used
# passed to `generate` directly to hot-fix cache issues, let's raise a warning instead of an error
# (otherwise a user might need to overwrite several parameters).
Expand All @@ -691,9 +703,9 @@ def validate(self, strict=False):
)

# 2.6. other incorrect combinations
if not self.return_dict_in_generate:
if self.return_dict_in_generate is not True:
for extra_output_flag in self.extra_output_flags:
if getattr(self, extra_output_flag):
if getattr(self, extra_output_flag) is True:
minor_issues[extra_output_flag] = (
f"`return_dict_in_generate` is NOT set to `True`, but `{extra_output_flag}` is. When "
f"`return_dict_in_generate` is not `True`, `{extra_output_flag}` is ignored."
Expand Down
14 changes: 0 additions & 14 deletions src/transformers/generation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1797,20 +1797,6 @@ def _prepare_generation_config(
generation_config.update(**self.generation_config.to_dict(), defaults_only=True)
generation_config.update(**global_defaults, defaults_only=True)

# Due to some values being boolean and not `None`, we need additional logic to overwrite
# them explicitly (`defaults_only=False`) on the condition that it's only a previous
# default value
default_generation_config = GenerationConfig()
generation_config.update(
**{
k: v
for k, v in self.generation_config.to_dict().items()
if isinstance(v, bool)
and hasattr(default_generation_config, k)
and getattr(generation_config, k, None) == getattr(default_generation_config, k)
}
)

# Finally, if there are any kwargs, update config with it -> highest priority at the end
model_kwargs = generation_config.update(**kwargs)

Expand Down
10 changes: 8 additions & 2 deletions src/transformers/models/clvp/modeling_clvp.py

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these changes really relevant? Seems unrelated to me

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

some tests were failing without it because the re-loaded model has a incorrect config type. And we couldn't find generation keys in it
It was luckily passing in main branch

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, then let's keep it, not wrong was just surprised

Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,8 @@ class ClvpEncoder(ClvpPreTrainedModel):
config: ClvpConfig
"""

config: ClvpEncoderConfig

def __init__(self, config: ClvpConfig):
super().__init__(config)

Expand Down Expand Up @@ -991,6 +993,8 @@ class ClvpDecoder(ClvpPreTrainedModel):
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ClvpDecoderLayer`]
"""

config: ClvpDecoderConfig

def __init__(self, config):
super().__init__(config)

Expand Down Expand Up @@ -1153,6 +1157,8 @@ def forward(

@auto_docstring
class ClvpModel(ClvpPreTrainedModel):
config: ClvpDecoderConfig

def __init__(self, config: ClvpDecoderConfig):
super().__init__(config)
self.config = config
Expand Down Expand Up @@ -1223,6 +1229,8 @@ def forward(
"""
)
class ClvpForCausalLM(ClvpPreTrainedModel, GenerationMixin):
config: ClvpDecoderConfig

def __init__(self, config):
super().__init__(config)

Expand Down Expand Up @@ -1418,8 +1426,6 @@ def forward(
"""
)
class ClvpModelForConditionalGeneration(ClvpPreTrainedModel, GenerationMixin):
config: ClvpConfig

def __init__(self, config: ClvpConfig):
super().__init__(config)

Expand Down
2 changes: 1 addition & 1 deletion tests/generation/test_configuration_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def test_kwarg_init(self):
"""Tests that we can overwrite attributes at `from_pretrained` time."""
default_config = GenerationConfig()
self.assertEqual(default_config.temperature, None)
self.assertEqual(default_config.do_sample, False)
self.assertEqual(default_config.do_sample, None)
self.assertEqual(default_config.num_beams, None)

config = GenerationConfig(
Expand Down
4 changes: 2 additions & 2 deletions tests/generation/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1207,7 +1207,7 @@ def test_generate_from_inputs_embeds_with_static_cache(self):

input_ids = inputs_dict.pop("input_ids")

model.config.use_cache = True
model.generation_config.use_cache = True
model.config.is_decoder = True
batch_size = input_ids.shape[0]
max_new_tokens = 10
Expand Down Expand Up @@ -1698,10 +1698,10 @@ def test_generate_methods_with_logits_to_keep(self):
self.skipTest(reason="This model does not support `logits_to_keep` argument.")

config, inputs_dict = self.prepare_config_and_inputs_for_generate()
config.use_cache = True
config.is_decoder = True

model = model_class(config).to(torch_device).eval()
model.generation_config.use_cache = True
# All generation methods (except assisted decoding) rely on always extracting the last token logits of the
# full logits matrix, so testing out only greedy search and assisted decoding is enough (if it works,
# other methods will work as well)
Expand Down
2 changes: 1 addition & 1 deletion tests/models/bart/test_modeling_bart.py
Original file line number Diff line number Diff line change
Expand Up @@ -987,7 +987,7 @@ def test_xsum_summarization_same_as_fairseq(self):

def test_xsum_config_generation_params(self):
model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-xsum")
expected_params = {"num_beams": 6, "do_sample": False, "early_stopping": True, "length_penalty": None}
expected_params = {"num_beams": 6, "do_sample": None, "early_stopping": True, "length_penalty": None}
config_params = {k: getattr(model.generation_config, k, "MISSING") for k, v in expected_params.items()}
self.assertDictEqual(expected_params, config_params)

Expand Down