From bfe637a60fcd4efcc0436784b391a08178c1178a Mon Sep 17 00:00:00 2001 From: tianmingxue Date: Wed, 5 Nov 2025 23:04:02 +0800 Subject: [PATCH 1/5] Add support for `num_generations_eval` parameter to allow different number of generations during evaluation. --- trl/trainer/grpo_config.py | 8 ++++++++ trl/trainer/grpo_trainer.py | 30 ++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index 6001a8dc524..3475fc41cfb 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -333,6 +333,14 @@ class GRPOConfig(TrainingArguments): "* gradient_accumulation_steps) must be evenly divisible by this value." }, ) + num_generations_eval: int | None = field( + default=None, + metadata={ + "help": "Number of generations to sample during evaluation. If `None`, uses the value of " + "`num_generations`. This allows using fewer generations during evaluation to save computation. " + "Maintains backward compatibility with previous configuration files." + }, + ) max_completion_length: int | None = field( default=256, metadata={"help": "Maximum length of the generated completion."}, diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 378dc2b6193..aceb1c906f8 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -380,6 +380,7 @@ def __init__( self.max_prompt_length = args.max_prompt_length self.max_completion_length = args.max_completion_length # = |o_i| in the GRPO paper self.num_generations = args.num_generations # = G in the GRPO paper + self.num_generations_eval = args.num_generations_eval self.chat_template_kwargs = args.chat_template_kwargs or {} self.temperature = args.temperature self.top_p = args.top_p @@ -758,9 +759,11 @@ def _get_train_sampler(self, dataset: Dataset | None = None) -> Sampler: def _get_eval_sampler(self, eval_dataset) -> Sampler: # See _get_train_sampler for an explanation of the sampler. + # If None, use num_generations for backward compatibility with previous config files + num_gens = self.num_generations_eval or self.num_generations return RepeatSampler( data_source=eval_dataset, - mini_repeat_count=self.num_generations, + mini_repeat_count=num_gens, seed=self.args.seed, ) @@ -1160,10 +1163,13 @@ def _generate_single_turn(self, prompts: list): # Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate # num_generations outputs for each one. This is faster than generating outputs for each duplicate # prompt individually. - ordered_set_of_prompts = all_prompts[:: self.num_generations] + # Determine num_generations based on mode + mode = "train" if self.model.training else "eval" + num_gens = self.num_generations_eval if mode == "eval" and self.num_generations_eval is not None else self.num_generations + ordered_set_of_prompts = all_prompts[:: num_gens] sampling_params = { - "n": self.num_generations, + "n": num_gens, "repetition_penalty": self.repetition_penalty, "temperature": self.temperature, "top_p": self.top_p, @@ -1210,8 +1216,12 @@ def _generate_single_turn(self, prompts: list): broadcast_object_list(obj_list, from_process=0) all_prompt_ids, all_completion_ids, all_logprobs, all_extra_fields = obj_list[0] + # Determine repeat count based on mode + mode = "train" if self.model.training else "eval" + num_gens = self.num_generations_eval if mode == "eval" and self.num_generations_eval is not None else self.num_generations # At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times - all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(self.num_generations)] + all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_gens)] + process_slice = slice( self.accelerator.process_index * len(prompts), @@ -1596,17 +1606,21 @@ def _generate_and_score_completions( # Apply weights to each reward function's output and sum rewards = (rewards_per_func * self.reward_weights.to(device).unsqueeze(0)).nansum(dim=1) + # If None, use num_generations for backward compatibility with previous config files + # Determine num_generations based on mode before computing grouped-wise rewards + mode = "train" if self.model.training else "eval" + num_gens = self.num_generations_eval or self.num_generations if mode == "eval" else self.num_generations # Compute grouped-wise rewards - mean_grouped_rewards = rewards.view(-1, self.num_generations).mean(dim=1) + mean_grouped_rewards = rewards.view(-1, num_gens).mean(dim=1) # Normalize the rewards to compute the advantages - mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(self.num_generations, dim=0) + mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_gens, dim=0) advantages = rewards - mean_grouped_rewards if self.scale_rewards in ["group", "none"]: # If self.scale_rewards = "none", we'll still log group level std - std_rewards = rewards.view(-1, self.num_generations).std(dim=1) - std_rewards = std_rewards.repeat_interleave(self.num_generations, dim=0) + std_rewards = rewards.view(-1, num_gens).std(dim=1) + std_rewards = std_rewards.repeat_interleave(num_gens, dim=0) elif self.scale_rewards == "batch": # Compute global std std_rewards = rewards.std().expand_as(rewards) From 4a1accd071b7768d182c382368812e065315f8fa Mon Sep 17 00:00:00 2001 From: tianmingxue Date: Wed, 5 Nov 2025 23:44:56 +0800 Subject: [PATCH 2/5] Add num_generations_eval parameter for efficient evaluation --- trl/trainer/grpo_trainer.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index aceb1c906f8..4433724223b 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -1165,8 +1165,12 @@ def _generate_single_turn(self, prompts: list): # prompt individually. # Determine num_generations based on mode mode = "train" if self.model.training else "eval" - num_gens = self.num_generations_eval if mode == "eval" and self.num_generations_eval is not None else self.num_generations - ordered_set_of_prompts = all_prompts[:: num_gens] + num_gens = ( + self.num_generations_eval + if mode == "eval" and self.num_generations_eval is not None + else self.num_generations + ) + ordered_set_of_prompts = all_prompts[::num_gens] sampling_params = { "n": num_gens, @@ -1218,10 +1222,13 @@ def _generate_single_turn(self, prompts: list): # Determine repeat count based on mode mode = "train" if self.model.training else "eval" - num_gens = self.num_generations_eval if mode == "eval" and self.num_generations_eval is not None else self.num_generations + num_gens = ( + self.num_generations_eval + if mode == "eval" and self.num_generations_eval is not None + else self.num_generations + ) # At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_gens)] - process_slice = slice( self.accelerator.process_index * len(prompts), From cbb264a386de7cb713484c220846321fb669f947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Nov 2025 02:22:50 +0000 Subject: [PATCH 3/5] same for rloo + doc --- trl/trainer/grpo_config.py | 14 ++++++++----- trl/trainer/grpo_trainer.py | 41 +++++++++++-------------------------- trl/trainer/rloo_config.py | 12 ++++++++++- trl/trainer/rloo_trainer.py | 16 +++++++++------ 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index b20a954a22f..f5c5af2cdf2 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -47,14 +47,18 @@ class GRPOConfig(TrainingArguments): is False. > Parameters that control the data preprocessing + remove_unused_columns (`bool`, *optional*, defaults to `False`): Whether to only keep the column `"prompt"` in the dataset. If you use a custom reward function that requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`. max_prompt_length (`int` or `None`, *optional*, defaults to `512`): Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left. - num_generations (`int` or `None`, *optional*, defaults to `8`): + num_generations (`int`, *optional*, defaults to `8`): Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size * gradient_accumulation_steps) must be evenly divisible by this value. + num_generations_eval (`int` or `None`, *optional*): + Number of generations to sample during evaluation. This allows using fewer generations during evaluation to + save computation. If `None`, uses the value of `num_generations`. max_completion_length (`int` or `None`, *optional*, defaults to `256`): Maximum length of the generated completion. ds3_gather_for_generation (`bool`, *optional*, defaults to `True`): @@ -363,9 +367,8 @@ class GRPOConfig(TrainingArguments): num_generations_eval: int | None = field( default=None, metadata={ - "help": "Number of generations to sample during evaluation. If `None`, uses the value of " - "`num_generations`. This allows using fewer generations during evaluation to save computation. " - "Maintains backward compatibility with previous configuration files." + "help": "Number of generations to sample during evaluation. This allows using fewer generations during " + "evaluation to save computation. If `None`, uses the value of `num_generations`." }, ) max_completion_length: int | None = field( @@ -703,7 +706,8 @@ class GRPOConfig(TrainingArguments): log_unique_prompts: bool = field( default=False, metadata={ - "help": "Whether to log unique prompts. If `True`, only unique prompts are logged. If `False`, all prompts are logged." + "help": "Whether to log unique prompts. If `True`, only unique prompts are logged. If `False`, all " + "prompts are logged." }, ) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 9aae7e61a8a..2195e5e28c0 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -379,7 +379,7 @@ def __init__( self.max_prompt_length = args.max_prompt_length self.max_completion_length = args.max_completion_length # = |o_i| in the GRPO paper self.num_generations = args.num_generations # = G in the GRPO paper - self.num_generations_eval = args.num_generations_eval + self.num_generations_eval = args.num_generations_eval or self.num_generations self.chat_template_kwargs = args.chat_template_kwargs or {} self.temperature = args.temperature self.top_p = args.top_p @@ -769,11 +769,9 @@ def _get_train_sampler(self, dataset: Dataset | None = None) -> Sampler: def _get_eval_sampler(self, eval_dataset) -> Sampler: # See _get_train_sampler for an explanation of the sampler. - # If None, use num_generations for backward compatibility with previous config files - num_gens = self.num_generations_eval or self.num_generations return RepeatSampler( data_source=eval_dataset, - mini_repeat_count=num_gens, + mini_repeat_count=self.num_generations_eval, seed=self.args.seed, ) @@ -1159,6 +1157,8 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list): def _generate_single_turn(self, prompts: list): device = self.accelerator.device + mode = "train" if self.model.training else "eval" + num_generations = self.num_generations if mode == "train" else self.num_generations_eval # Generate completions using either vLLM or regular generation if self.use_vllm: @@ -1183,17 +1183,10 @@ def _generate_single_turn(self, prompts: list): # Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate # num_generations outputs for each one. This is faster than generating outputs for each duplicate # prompt individually. - # Determine num_generations based on mode - mode = "train" if self.model.training else "eval" - num_gens = ( - self.num_generations_eval - if mode == "eval" and self.num_generations_eval is not None - else self.num_generations - ) - ordered_set_of_prompts = all_prompts[::num_gens] + ordered_set_of_prompts = all_prompts[::num_generations] sampling_params = { - "n": num_gens, + "n": num_generations, "repetition_penalty": self.repetition_penalty, "temperature": self.temperature, "top_p": self.top_p, @@ -1236,15 +1229,8 @@ def _generate_single_turn(self, prompts: list): broadcast_object_list(obj_list, from_process=0) all_prompt_ids, all_completion_ids, all_logprobs, all_extra_fields = obj_list[0] - # Determine repeat count based on mode - mode = "train" if self.model.training else "eval" - num_gens = ( - self.num_generations_eval - if mode == "eval" and self.num_generations_eval is not None - else self.num_generations - ) # At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times - all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_gens)] + all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_generations)] process_slice = slice( self.accelerator.process_index * len(prompts), @@ -1650,21 +1636,18 @@ def _generate_and_score_completions( # Apply weights to each reward function's output and sum rewards = (rewards_per_func * self.reward_weights.to(device).unsqueeze(0)).nansum(dim=1) - # If None, use num_generations for backward compatibility with previous config files - # Determine num_generations based on mode before computing grouped-wise rewards - mode = "train" if self.model.training else "eval" - num_gens = self.num_generations_eval or self.num_generations if mode == "eval" else self.num_generations # Compute grouped-wise rewards - mean_grouped_rewards = rewards.view(-1, num_gens).mean(dim=1) + num_generations = self.num_generations if mode == "train" else self.num_generations_eval + mean_grouped_rewards = rewards.view(-1, num_generations).mean(dim=1) # Normalize the rewards to compute the advantages - mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_gens, dim=0) + mean_grouped_rewards = mean_grouped_rewards.repeat_interleave(num_generations, dim=0) advantages = rewards - mean_grouped_rewards if self.scale_rewards in ["group", "none"]: # If self.scale_rewards = "none", we'll still log group level std - std_rewards = rewards.view(-1, num_gens).std(dim=1) - std_rewards = std_rewards.repeat_interleave(num_gens, dim=0) + std_rewards = rewards.view(-1, num_generations).std(dim=1) + std_rewards = std_rewards.repeat_interleave(num_generations, dim=0) elif self.scale_rewards == "batch": # Compute global std std_rewards = rewards.std().expand_as(rewards) diff --git a/trl/trainer/rloo_config.py b/trl/trainer/rloo_config.py index eb893c604c8..d615fd15d93 100644 --- a/trl/trainer/rloo_config.py +++ b/trl/trainer/rloo_config.py @@ -48,9 +48,12 @@ class RLOOConfig(TrainingArguments): requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`. max_prompt_length (`int` or `None`, *optional*, defaults to `512`): Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left. - num_generations (`int` or `None`, *optional*, defaults to `2`): + num_generations (`int`, *optional*, defaults to `8`): Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size * gradient_accumulation_steps) must be evenly divisible by this value. + num_generations_eval (`int` or `None`, *optional*): + Number of generations to sample during evaluation. This allows using fewer generations during evaluation to + save computation. If `None`, uses the value of `num_generations`. max_completion_length (`int` or `None`, *optional*, defaults to `256`): Maximum length of the generated completion. ds3_gather_for_generation (`bool`, *optional*, defaults to `True`): @@ -286,6 +289,13 @@ class RLOOConfig(TrainingArguments): "* gradient_accumulation_steps) must be evenly divisible by this value." }, ) + num_generations_eval: int | None = field( + default=None, + metadata={ + "help": "Number of generations to sample during evaluation. This allows using fewer generations during " + "evaluation to save computation. If `None`, uses the value of `num_generations`." + }, + ) max_completion_length: int | None = field( default=256, metadata={"help": "Maximum length of the generated completion."}, diff --git a/trl/trainer/rloo_trainer.py b/trl/trainer/rloo_trainer.py index ca572caae7e..166991e99c2 100644 --- a/trl/trainer/rloo_trainer.py +++ b/trl/trainer/rloo_trainer.py @@ -370,6 +370,7 @@ def __init__( self.max_prompt_length = args.max_prompt_length self.max_completion_length = args.max_completion_length self.num_generations = args.num_generations + self.num_generations_eval = args.num_generations_eval or args.num_generations self.chat_template_kwargs = args.chat_template_kwargs or {} self.temperature = args.temperature self.top_p = args.top_p @@ -691,7 +692,7 @@ def _get_eval_sampler(self, eval_dataset) -> Sampler: # See _get_train_sampler for an explanation of the sampler. return RepeatSampler( data_source=eval_dataset, - mini_repeat_count=self.num_generations, + mini_repeat_count=self.num_generations_eval, seed=self.args.seed, ) @@ -997,6 +998,8 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list): def _generate_single_turn(self, prompts: list): device = self.accelerator.device + mode = "train" if self.model.training else "eval" + num_generations = self.num_generations if mode == "train" else self.num_generations_eval # Generate completions using either vLLM or regular generation if self.use_vllm: @@ -1020,10 +1023,10 @@ def _generate_single_turn(self, prompts: list): # Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate # num_generations outputs for each one. This is faster than generating outputs for each duplicate # prompt individually. - ordered_set_of_prompts = all_prompts[:: self.num_generations] + ordered_set_of_prompts = all_prompts[:: num_generations] sampling_params = { - "n": self.num_generations, + "n": num_generations, "repetition_penalty": self.repetition_penalty, "temperature": self.temperature, "top_p": self.top_p, @@ -1053,7 +1056,7 @@ def _generate_single_turn(self, prompts: list): all_prompt_ids, all_completion_ids, _ = obj_list[0] # At this point, we only get 1 copy of each prompt, so we need to repeat them num_generations times - all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(self.num_generations)] + all_prompt_ids = [ids for ids in all_prompt_ids for _ in range(num_generations)] process_slice = slice( self.accelerator.process_index * len(prompts), @@ -1395,14 +1398,15 @@ def _generate_and_score_completions( kl = gather(kl) # rewards are gathered, so kl must be too rewards = rewards - self.beta * kl - grouped_rewards = rewards.view(-1, self.num_generations) + num_generations = self.num_generations if mode == "train" else self.num_generations_eval + grouped_rewards = rewards.view(-1, num_generations) mean_grouped_rewards = grouped_rewards.mean(dim=1) std_rewards = grouped_rewards.std(dim=1) is_std_zero = torch.isclose(std_rewards, torch.zeros_like(std_rewards)) # RLOO advantages computation grouped_sum = grouped_rewards.sum(dim=1, keepdim=True) # (num_prompts, 1) - baselines = (grouped_sum - grouped_rewards) / (self.num_generations - 1) # (num_prompts, num_generations) + baselines = (grouped_sum - grouped_rewards) / (num_generations - 1) # (num_prompts, num_generations) baselines = baselines.view(-1) # Flatten back to match rewards shape advantages = rewards - baselines From 9ab53392388c835fcb66df400aa56553882af148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Nov 2025 02:25:52 +0000 Subject: [PATCH 4/5] revert unwanted change --- trl/trainer/rloo_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/rloo_config.py b/trl/trainer/rloo_config.py index d615fd15d93..32f768f550f 100644 --- a/trl/trainer/rloo_config.py +++ b/trl/trainer/rloo_config.py @@ -48,7 +48,7 @@ class RLOOConfig(TrainingArguments): requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`. max_prompt_length (`int` or `None`, *optional*, defaults to `512`): Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left. - num_generations (`int`, *optional*, defaults to `8`): + num_generations (`int`, *optional*, defaults to `2`): Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size * gradient_accumulation_steps) must be evenly divisible by this value. num_generations_eval (`int` or `None`, *optional*): From 92d45155aaadc1ba93a5d08627f0aa27fa0e0de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Nov 2025 02:50:59 +0000 Subject: [PATCH 5/5] test --- tests/test_grpo_trainer.py | 24 ++++++++++++++++++++++++ tests/test_rloo_trainer.py | 24 ++++++++++++++++++++++++ trl/trainer/grpo_trainer.py | 2 +- trl/trainer/rloo_trainer.py | 4 ++-- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tests/test_grpo_trainer.py b/tests/test_grpo_trainer.py index b3844a399c1..ffe482d74f9 100644 --- a/tests/test_grpo_trainer.py +++ b/tests/test_grpo_trainer.py @@ -223,6 +223,30 @@ def test_training_with_eval(self): trainer.train() + def test_training_with_num_generations_eval(self): + dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only") + + training_args = GRPOConfig( + output_dir=self.tmp_dir, + per_device_train_batch_size=3, # reduce the batch size to reduce memory usage + per_device_eval_batch_size=3, # reduce the batch size to reduce memory usage + num_generations=3, # reduce the number of generations to reduce memory usage + max_completion_length=8, # reduce the completion length to reduce memory usage + num_generations_eval=1, + eval_strategy="steps", + eval_steps=2, + report_to="none", + ) + trainer = GRPOTrainer( + model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5", + reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5", + args=training_args, + train_dataset=dataset["train"], + eval_dataset=dataset["test"], + ) + + trainer.train() + def test_training_multiple_iterations(self): dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train") diff --git a/tests/test_rloo_trainer.py b/tests/test_rloo_trainer.py index 82810b1cca5..9eb0eeee13d 100644 --- a/tests/test_rloo_trainer.py +++ b/tests/test_rloo_trainer.py @@ -97,6 +97,30 @@ def test_training_with_eval(self): trainer.train() + def test_training_with_num_generations_eval(self): + dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only") + + training_args = RLOOConfig( + output_dir=self.tmp_dir, + per_device_train_batch_size=3, # reduce the batch size to reduce memory usage + per_device_eval_batch_size=3, # reduce the batch size to reduce memory usage + num_generations=3, # reduce the number of generations to reduce memory usage + max_completion_length=8, # reduce the completion length to reduce memory usage + num_generations_eval=1, + eval_strategy="steps", + eval_steps=2, + report_to="none", + ) + trainer = RLOOTrainer( + model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5", + reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5", + args=training_args, + train_dataset=dataset["train"], + eval_dataset=dataset["test"], + ) + + trainer.train() + def test_training_multiple_iterations(self): dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train") diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 2195e5e28c0..47588b4d024 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -1158,7 +1158,6 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list): def _generate_single_turn(self, prompts: list): device = self.accelerator.device mode = "train" if self.model.training else "eval" - num_generations = self.num_generations if mode == "train" else self.num_generations_eval # Generate completions using either vLLM or regular generation if self.use_vllm: @@ -1178,6 +1177,7 @@ def _generate_single_turn(self, prompts: list): # Generate completions using vLLM: gather all prompts and use them in a single call in the main process if self.vllm_mode == "server": all_prompts = gather_object(prompts) + num_generations = self.num_generations if mode == "train" else self.num_generations_eval if self.accelerator.is_main_process: # Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate diff --git a/trl/trainer/rloo_trainer.py b/trl/trainer/rloo_trainer.py index 166991e99c2..b24b8b59b92 100644 --- a/trl/trainer/rloo_trainer.py +++ b/trl/trainer/rloo_trainer.py @@ -999,7 +999,6 @@ def _calculate_rewards(self, inputs, prompts, completions, completion_ids_list): def _generate_single_turn(self, prompts: list): device = self.accelerator.device mode = "train" if self.model.training else "eval" - num_generations = self.num_generations if mode == "train" else self.num_generations_eval # Generate completions using either vLLM or regular generation if self.use_vllm: @@ -1018,12 +1017,13 @@ def _generate_single_turn(self, prompts: list): # Generate completions using vLLM: gather all prompts and use them in a single call in the main process if self.vllm_mode == "server": all_prompts = gather_object(prompts) + num_generations = self.num_generations if mode == "train" else self.num_generations_eval if self.accelerator.is_main_process: # Since 'prompts' contains 'num_generations' duplicates, we first take unique prompts, and generate # num_generations outputs for each one. This is faster than generating outputs for each duplicate # prompt individually. - ordered_set_of_prompts = all_prompts[:: num_generations] + ordered_set_of_prompts = all_prompts[::num_generations] sampling_params = { "n": num_generations,