Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions trl/trainer/grpo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,14 @@ class GRPOConfig(TrainingArguments):
"* gradient_accumulation_steps) must be evenly divisible by this value."
},
)
num_generations_eval: int | None = field(
default=None,
metadata={
"help": "Number of generations to sample during evaluation. If `None`, uses the value of "
"`num_generations`. This allows using fewer generations during evaluation to save computation. "
"Maintains backward compatibility with previous configuration files."
},
)
max_completion_length: int | None = field(
default=256,
metadata={"help": "Maximum length of the generated completion."},
Expand Down
37 changes: 29 additions & 8 deletions trl/trainer/grpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ def __init__(
self.max_prompt_length = args.max_prompt_length
self.max_completion_length = args.max_completion_length # = |o_i| in the GRPO paper
self.num_generations = args.num_generations # = G in the GRPO paper
self.num_generations_eval = args.num_generations_eval
self.chat_template_kwargs = args.chat_template_kwargs or {}
self.temperature = args.temperature
self.top_p = args.top_p
Expand Down Expand Up @@ -758,9 +759,11 @@ def _get_train_sampler(self, dataset: Dataset | None = None) -> Sampler:

def _get_eval_sampler(self, eval_dataset) -> Sampler:
# See _get_train_sampler for an explanation of the sampler.
# If None, use num_generations for backward compatibility with previous config files
num_gens = self.num_generations_eval or self.num_generations
return RepeatSampler(
data_source=eval_dataset,
mini_repeat_count=self.num_generations,
mini_repeat_count=num_gens,
seed=self.args.seed,
)

Expand Down Expand Up @@ -1160,10 +1163,17 @@ def _generate_single_turn(self, prompts: list):
# Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate
# num_generations outputs for each one. This is faster than generating outputs for each duplicate
# prompt individually.
ordered_set_of_prompts = all_prompts[:: self.num_generations]
# Determine num_generations based on mode
mode = "train" if self.model.training else "eval"
num_gens = (
self.num_generations_eval
if mode == "eval" and self.num_generations_eval is not None
else self.num_generations
)
ordered_set_of_prompts = all_prompts[::num_gens]

sampling_params = {
"n": self.num_generations,
"n": num_gens,
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
Expand Down Expand Up @@ -1210,8 +1220,15 @@ def _generate_single_turn(self, prompts: list):
broadcast_object_list(obj_list, from_process=0)
all_prompt_ids, all_completion_ids, all_logprobs, all_extra_fields = obj_list[0]

# Determine repeat count based on mode
mode = "train" if self.model.training else "eval"
num_gens = (
self.num_generations_eval
if mode == "eval" and self.num_generations_eval is not None
else self.num_generations
)
# At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(self.num_generations)]
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_gens)]

process_slice = slice(
self.accelerator.process_index * len(prompts),
Expand Down Expand Up @@ -1596,17 +1613,21 @@ def _generate_and_score_completions(
# Apply weights to each reward function's output and sum
rewards = (rewards_per_func * self.reward_weights.to(device).unsqueeze(0)).nansum(dim=1)

# If None, use num_generations for backward compatibility with previous config files
# Determine num_generations based on mode before computing grouped-wise rewards
mode = "train" if self.model.training else "eval"
num_gens = self.num_generations_eval or self.num_generations if mode == "eval" else self.num_generations
# Compute grouped-wise rewards
mean_grouped_rewards = rewards.view(-1, self.num_generations).mean(dim=1)
mean_grouped_rewards = rewards.view(-1, num_gens).mean(dim=1)

# Normalize the rewards to compute the advantages
mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(self.num_generations, dim=0)
mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_gens, dim=0)
advantages = rewards - mean_grouped_rewards

if self.scale_rewards in ["group", "none"]:
# If self.scale_rewards = "none", we'll still log group level std
std_rewards = rewards.view(-1, self.num_generations).std(dim=1)
std_rewards = std_rewards.repeat_interleave(self.num_generations, dim=0)
std_rewards = rewards.view(-1, num_gens).std(dim=1)
std_rewards = std_rewards.repeat_interleave(num_gens, dim=0)
elif self.scale_rewards == "batch":
# Compute global std
std_rewards = rewards.std().expand_as(rewards)
Expand Down