Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion trl/experimental/gold/gold_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,10 @@ class GOLDConfig(SFTConfig):
)
top_k: int = field(
default=0,
metadata={"help": "The number of highest probability vocabulary tokens to keep for top-k-filtering."},
metadata={
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, "
"top-k-filtering is disabled and all tokens are considered."
},
)
lmbda: float = field(
default=0.5,
Expand Down
10 changes: 5 additions & 5 deletions trl/experimental/online_dpo/online_dpo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ class may differ from those in [`~transformers.TrainingArguments`].
top_p (`float`, *optional*, defaults to `1.0`):
Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to
`1.0` to consider all tokens.
top_k (`int`, *optional*):
Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, top-k-filtering is
top_k (`int`, defaults to `0`):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
top_k (`int`, defaults to `0`):
top_k (`int`, *optional*, defaults to `0`):

Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, top-k-filtering is
disabled and all tokens are considered.
min_p (`float`, *optional*):
Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
Expand Down Expand Up @@ -220,10 +220,10 @@ class may differ from those in [`~transformers.TrainingArguments`].
"Set to 1.0 to consider all tokens."
},
)
top_k: int | None = field(
default=None,
top_k: int = field(
default=0,
metadata={
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, "
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, "
"top-k-filtering is disabled and all tokens are considered."
},
)
Expand Down
2 changes: 1 addition & 1 deletion trl/experimental/online_dpo/online_dpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ def __init__(
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": -1 if self.top_k is None else self.top_k,
"top_k": -1 if not self.top_k else self.top_k,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the vLLM support to consider all tokens by setting top_k=0 (besides -1) was introduced in vllm-0.9.0

and we require vllm>=0.10.2.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On the other hand, what if the users still set top_k=None? Maybe we should start a deprecation cycle.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could indeed deprecate None for 1 version before removal. I'd do it only for GRPO and RLOO though

"min_p": 0.0 if self.min_p is None else self.min_p,
"max_tokens": args.max_new_tokens,
"detokenize": False, # to avoid vllm to decode (we don't need it)
Expand Down
2 changes: 1 addition & 1 deletion trl/experimental/openenv/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def _build_colocate_sampling_params(
generation_kwargs: dict[str, Any] = {
"n": 1,
"temperature": trainer.temperature,
"top_k": -1 if trainer.top_k is None else trainer.top_k,
"top_k": -1 if not trainer.top_k else trainer.top_k,
"min_p": 0.0 if trainer.min_p is None else trainer.min_p,
"max_tokens": trainer.max_completion_length,
"guided_decoding": guided_decoding,
Expand Down
10 changes: 5 additions & 5 deletions trl/trainer/grpo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ class GRPOConfig(TrainingArguments):
top_p (`float`, *optional*, defaults to `1.0`):
Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to
`1.0` to consider all tokens.
top_k (`int`, *optional*):
Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, top-k-filtering is
top_k (`int`, defaults to `0`):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
top_k (`int`, defaults to `0`):
top_k (`int`, *optional*, defaults to `0`):

Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, top-k-filtering is
disabled and all tokens are considered.
min_p (`float`, *optional*):
Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
Expand Down Expand Up @@ -433,10 +433,10 @@ class GRPOConfig(TrainingArguments):
"Set to 1.0 to consider all tokens."
},
)
top_k: int | None = field(
default=None,
top_k: int = field(
default=0,
metadata={
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, "
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, "
"top-k-filtering is disabled and all tokens are considered."
},
)
Expand Down
4 changes: 2 additions & 2 deletions trl/trainer/grpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1270,7 +1270,7 @@ def _generate_single_turn(self, prompts: list):
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": -1 if self.top_k is None else self.top_k,
"top_k": -1 if not self.top_k else self.top_k,
"min_p": 0.0 if self.min_p is None else self.min_p,
"max_tokens": self.max_completion_length,
"guided_decoding_regex": self.guided_decoding_regex,
Expand Down Expand Up @@ -1357,7 +1357,7 @@ def _generate_single_turn(self, prompts: list):
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": -1 if self.top_k is None else self.top_k,
"top_k": -1 if not self.top_k else self.top_k,
"min_p": 0.0 if self.min_p is None else self.min_p,
"max_tokens": self.max_completion_length,
"guided_decoding": guided_decoding,
Expand Down
10 changes: 5 additions & 5 deletions trl/trainer/rloo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ class RLOOConfig(TrainingArguments):
top_p (`float`, *optional*, defaults to `1.0`):
Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to
`1.0` to consider all tokens.
top_k (`int`, *optional*):
Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, top-k-filtering is
top_k (`int`, defaults to `0`):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
top_k (`int`, defaults to `0`):
top_k (`int`, *optional*, defaults to `0`):

Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, top-k-filtering is
disabled and all tokens are considered.
min_p (`float`, *optional*):
Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
Expand Down Expand Up @@ -328,10 +328,10 @@ class RLOOConfig(TrainingArguments):
"Set to 1.0 to consider all tokens."
},
)
top_k: int | None = field(
default=None,
top_k: int = field(
default=0,
metadata={
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, "
"help": "Number of highest probability vocabulary tokens to keep for top-k-filtering. If `0`, "
"top-k-filtering is disabled and all tokens are considered."
},
)
Expand Down
4 changes: 2 additions & 2 deletions trl/trainer/rloo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1037,7 +1037,7 @@ def _generate_single_turn(self, prompts: list):
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": -1 if self.top_k is None else self.top_k,
"top_k": -1 if not self.top_k else self.top_k,
"min_p": 0.0 if self.min_p is None else self.min_p,
"max_tokens": self.max_completion_length,
"truncate_prompt_tokens": self.max_prompt_length,
Expand Down Expand Up @@ -1084,7 +1084,7 @@ def _generate_single_turn(self, prompts: list):
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": -1 if self.top_k is None else self.top_k,
"top_k": -1 if not self.top_k else self.top_k,
"min_p": 0.0 if self.min_p is None else self.min_p,
"max_tokens": self.max_completion_length,
"truncate_prompt_tokens": self.max_prompt_length,
Expand Down
Loading