Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions tests/test_grpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,30 @@ def test_training_with_eval(self):

trainer.train()

def test_training_with_num_generations_eval(self):
dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only")

training_args = GRPOConfig(
output_dir=self.tmp_dir,
per_device_train_batch_size=3, # reduce the batch size to reduce memory usage
per_device_eval_batch_size=3, # reduce the batch size to reduce memory usage
num_generations=3, # reduce the number of generations to reduce memory usage
max_completion_length=8, # reduce the completion length to reduce memory usage
num_generations_eval=1,
eval_strategy="steps",
eval_steps=2,
report_to="none",
)
trainer = GRPOTrainer(
model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5",
reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5",
args=training_args,
train_dataset=dataset["train"],
eval_dataset=dataset["test"],
)

trainer.train()

def test_training_multiple_iterations(self):
dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train")

Expand Down
24 changes: 24 additions & 0 deletions tests/test_rloo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,30 @@ def test_training_with_eval(self):

trainer.train()

def test_training_with_num_generations_eval(self):
dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only")

training_args = RLOOConfig(
output_dir=self.tmp_dir,
per_device_train_batch_size=3, # reduce the batch size to reduce memory usage
per_device_eval_batch_size=3, # reduce the batch size to reduce memory usage
num_generations=3, # reduce the number of generations to reduce memory usage
max_completion_length=8, # reduce the completion length to reduce memory usage
num_generations_eval=1,
eval_strategy="steps",
eval_steps=2,
report_to="none",
)
trainer = RLOOTrainer(
model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5",
reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5",
args=training_args,
train_dataset=dataset["train"],
eval_dataset=dataset["test"],
)

trainer.train()

def test_training_multiple_iterations(self):
dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train")

Expand Down
16 changes: 14 additions & 2 deletions trl/trainer/grpo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,18 @@ class GRPOConfig(TrainingArguments):
is False.

> Parameters that control the data preprocessing

remove_unused_columns (`bool`, *optional*, defaults to `False`):
Whether to only keep the column `"prompt"` in the dataset. If you use a custom reward function that
requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`.
max_prompt_length (`int` or `None`, *optional*, defaults to `512`):
Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left.
num_generations (`int` or `None`, *optional*, defaults to `8`):
num_generations (`int`, *optional*, defaults to `8`):
Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size
* gradient_accumulation_steps) must be evenly divisible by this value.
num_generations_eval (`int` or `None`, *optional*):
Number of generations to sample during evaluation. This allows using fewer generations during evaluation to
save computation. If `None`, uses the value of `num_generations`.
max_completion_length (`int` or `None`, *optional*, defaults to `256`):
Maximum length of the generated completion.
ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):
Expand Down Expand Up @@ -360,6 +364,13 @@ class GRPOConfig(TrainingArguments):
"* gradient_accumulation_steps) must be evenly divisible by this value."
},
)
num_generations_eval: int | None = field(
default=None,
metadata={
"help": "Number of generations to sample during evaluation. This allows using fewer generations during "
"evaluation to save computation. If `None`, uses the value of `num_generations`."
},
)
max_completion_length: int | None = field(
default=256,
metadata={"help": "Maximum length of the generated completion."},
Expand Down Expand Up @@ -695,7 +706,8 @@ class GRPOConfig(TrainingArguments):
log_unique_prompts: bool = field(
default=False,
metadata={
"help": "Whether to log unique prompts. If `True`, only unique prompts are logged. If `False`, all prompts are logged."
"help": "Whether to log unique prompts. If `True`, only unique prompts are logged. If `False`, all "
"prompts are logged."
},
)

Expand Down
20 changes: 12 additions & 8 deletions trl/trainer/grpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,7 @@ def __init__(
self.max_prompt_length = args.max_prompt_length
self.max_completion_length = args.max_completion_length # = |o_i| in the GRPO paper
self.num_generations = args.num_generations # = G in the GRPO paper
self.num_generations_eval = args.num_generations_eval or self.num_generations
self.chat_template_kwargs = args.chat_template_kwargs or {}
self.temperature = args.temperature
self.top_p = args.top_p
Expand Down Expand Up @@ -770,7 +771,7 @@ def _get_eval_sampler(self, eval_dataset) -> Sampler:
# See _get_train_sampler for an explanation of the sampler.
return RepeatSampler(
data_source=eval_dataset,
mini_repeat_count=self.num_generations,
mini_repeat_count=self.num_generations_eval,
seed=self.args.seed,
)

Expand Down Expand Up @@ -1156,6 +1157,7 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list):

def _generate_single_turn(self, prompts: list):
device = self.accelerator.device
mode = "train" if self.model.training else "eval"

# Generate completions using either vLLM or regular generation
if self.use_vllm:
Expand All @@ -1175,15 +1177,16 @@ def _generate_single_turn(self, prompts: list):
# Generate completions using vLLM: gather all prompts and use them in a single call in the main process
if self.vllm_mode == "server":
all_prompts = gather_object(prompts)
num_generations = self.num_generations if mode == "train" else self.num_generations_eval

if self.accelerator.is_main_process:
# Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate
# num_generations outputs for each one. This is faster than generating outputs for each duplicate
# prompt individually.
ordered_set_of_prompts = all_prompts[:: self.num_generations]
ordered_set_of_prompts = all_prompts[::num_generations]

sampling_params = {
"n": self.num_generations,
"n": num_generations,
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
Expand Down Expand Up @@ -1227,7 +1230,7 @@ def _generate_single_turn(self, prompts: list):
all_prompt_ids, all_completion_ids, all_logprobs, all_extra_fields = obj_list[0]

# At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(self.num_generations)]
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_generations)]

process_slice = slice(
self.accelerator.process_index * len(prompts),
Expand Down Expand Up @@ -1634,16 +1637,17 @@ def _generate_and_score_completions(
rewards = (rewards_per_func * self.reward_weights.to(device).unsqueeze(0)).nansum(dim=1)

# Compute grouped-wise rewards
mean_grouped_rewards = rewards.view(-1, self.num_generations).mean(dim=1)
num_generations = self.num_generations if mode == "train" else self.num_generations_eval
mean_grouped_rewards = rewards.view(-1, num_generations).mean(dim=1)

# Normalize the rewards to compute the advantages
mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(self.num_generations, dim=0)
mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_generations, dim=0)
advantages = rewards - mean_grouped_rewards

if self.scale_rewards in ["group", "none"]:
# If self.scale_rewards = "none", we'll still log group level std
std_rewards = rewards.view(-1, self.num_generations).std(dim=1)
std_rewards = std_rewards.repeat_interleave(self.num_generations, dim=0)
std_rewards = rewards.view(-1, num_generations).std(dim=1)
std_rewards = std_rewards.repeat_interleave(num_generations, dim=0)
elif self.scale_rewards == "batch":
# Compute global std
std_rewards = rewards.std().expand_as(rewards)
Expand Down
12 changes: 11 additions & 1 deletion trl/trainer/rloo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,12 @@ class RLOOConfig(TrainingArguments):
requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`.
max_prompt_length (`int` or `None`, *optional*, defaults to `512`):
Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left.
num_generations (`int` or `None`, *optional*, defaults to `2`):
num_generations (`int`, *optional*, defaults to `2`):
Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size
* gradient_accumulation_steps) must be evenly divisible by this value.
num_generations_eval (`int` or `None`, *optional*):
Number of generations to sample during evaluation. This allows using fewer generations during evaluation to
save computation. If `None`, uses the value of `num_generations`.
max_completion_length (`int` or `None`, *optional*, defaults to `256`):
Maximum length of the generated completion.
ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):
Expand Down Expand Up @@ -286,6 +289,13 @@ class RLOOConfig(TrainingArguments):
"* gradient_accumulation_steps) must be evenly divisible by this value."
},
)
num_generations_eval: int | None = field(
default=None,
metadata={
"help": "Number of generations to sample during evaluation. This allows using fewer generations during "
"evaluation to save computation. If `None`, uses the value of `num_generations`."
},
)
max_completion_length: int | None = field(
default=256,
metadata={"help": "Maximum length of the generated completion."},
Expand Down
16 changes: 10 additions & 6 deletions trl/trainer/rloo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ def __init__(
self.max_prompt_length = args.max_prompt_length
self.max_completion_length = args.max_completion_length
self.num_generations = args.num_generations
self.num_generations_eval = args.num_generations_eval or args.num_generations
self.chat_template_kwargs = args.chat_template_kwargs or {}
self.temperature = args.temperature
self.top_p = args.top_p
Expand Down Expand Up @@ -691,7 +692,7 @@ def _get_eval_sampler(self, eval_dataset) -> Sampler:
# See _get_train_sampler for an explanation of the sampler.
return RepeatSampler(
data_source=eval_dataset,
mini_repeat_count=self.num_generations,
mini_repeat_count=self.num_generations_eval,
seed=self.args.seed,
)

Expand Down Expand Up @@ -997,6 +998,7 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list):

def _generate_single_turn(self, prompts: list):
device = self.accelerator.device
mode = "train" if self.model.training else "eval"

# Generate completions using either vLLM or regular generation
if self.use_vllm:
Expand All @@ -1015,15 +1017,16 @@ def _generate_single_turn(self, prompts: list):
# Generate completions using vLLM: gather all prompts and use them in a single call in the main process
if self.vllm_mode == "server":
all_prompts = gather_object(prompts)
num_generations = self.num_generations if mode == "train" else self.num_generations_eval

if self.accelerator.is_main_process:
# Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate
# num_generations outputs for each one. This is faster than generating outputs for each duplicate
# prompt individually.
ordered_set_of_prompts = all_prompts[:: self.num_generations]
ordered_set_of_prompts = all_prompts[::num_generations]

sampling_params = {
"n": self.num_generations,
"n": num_generations,
"repetition_penalty": self.repetition_penalty,
"temperature": self.temperature,
"top_p": self.top_p,
Expand Down Expand Up @@ -1053,7 +1056,7 @@ def _generate_single_turn(self, prompts: list):
all_prompt_ids, all_completion_ids, _ = obj_list[0]

# At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(self.num_generations)]
all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_generations)]

process_slice = slice(
self.accelerator.process_index * len(prompts),
Expand Down Expand Up @@ -1395,14 +1398,15 @@ def _generate_and_score_completions(
kl = gather(kl) # rewards are gathered, so kl must be too
rewards = rewards - self.beta * kl

grouped_rewards = rewards.view(-1, self.num_generations)
num_generations = self.num_generations if mode == "train" else self.num_generations_eval
grouped_rewards = rewards.view(-1, num_generations)
mean_grouped_rewards = grouped_rewards.mean(dim=1)
std_rewards = grouped_rewards.std(dim=1)
is_std_zero = torch.isclose(std_rewards, torch.zeros_like(std_rewards))

# RLOO advantages computation
grouped_sum = grouped_rewards.sum(dim=1, keepdim=True) # (num_prompts, 1)
baselines = (grouped_sum - grouped_rewards) / (self.num_generations - 1) # (num_prompts, num_generations)
baselines = (grouped_sum - grouped_rewards) / (num_generations - 1) # (num_prompts, num_generations)
baselines = baselines.view(-1) # Flatten back to match rewards shape
advantages = rewards - baselines

Expand Down