Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions trl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@
"KTOTrainer",
"LogCompletionsCallback",
"ModelConfig",
"MiniLLMConfig",
"MiniLLMTrainer",
"NashMDConfig",
"NashMDTrainer",
"OnlineDPOConfig",
Expand Down Expand Up @@ -155,6 +157,8 @@
KTOTrainer,
LogCompletionsCallback,
ModelConfig,
MiniLLMConfig,
MiniLLMTrainer,
NashMDConfig,
NashMDTrainer,
OnlineDPOConfig,
Expand Down
4 changes: 4 additions & 0 deletions trl/trainer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
"kto_config": ["KTOConfig"],
"kto_trainer": ["KTOTrainer"],
"model_config": ["ModelConfig"],
"minillm_config": ["MiniLLMConfig"],
"minillm_trainer": ["MiniLLMTrainer"],
"nash_md_config": ["NashMDConfig"],
"nash_md_trainer": ["NashMDTrainer"],
"online_dpo_config": ["OnlineDPOConfig"],
Expand Down Expand Up @@ -110,6 +112,8 @@
from .kto_config import KTOConfig
from .kto_trainer import KTOTrainer
from .model_config import ModelConfig
from .minillm_config import MiniLLMConfig
from .minillm_trainer import MiniLLMTrainer
from .nash_md_config import NashMDConfig
from .nash_md_trainer import NashMDTrainer
from .online_dpo_config import OnlineDPOConfig
Expand Down
14 changes: 8 additions & 6 deletions trl/trainer/grpo_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1800,6 +1800,8 @@ def _compute_loss(self, model, inputs):

# Compute the loss
advantages = inputs["advantages"]
if advantages.dim() == 1:
advantages = advantages.unsqueeze(1)
# When num_iterations == 1 and steps_per_generation <= gradient_accumulation_steps,
# old_per_token_logps == per_token_logps. In this case we can skip its computation
# (see _generate_and_score_completions) and instead use per_token_logps.detach().
Expand All @@ -1826,15 +1828,15 @@ def _compute_loss(self, model, inputs):
# importance_sampling_level: "token" level: (B, T); "sequence" level: (B, 1)
if self.loss_type == "cispo":
clamped_ratios = torch.clamp(coef_1, max=self.epsilon_high).detach()
per_token_loss = -clamped_ratios * advantages.unsqueeze(1) * per_token_logps
per_token_loss = -clamped_ratios * advantages * per_token_logps
elif self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]:
coef_2 = torch.clamp(coef_1, 1 - self.epsilon_low, 1 + self.epsilon_high)
# Two-sided clipping
if self.args.delta is not None:
coef_1 = torch.clamp(coef_1, max=self.args.delta)

per_token_loss1 = coef_1 * advantages.unsqueeze(1)
per_token_loss2 = coef_2 * advantages.unsqueeze(1)
per_token_loss1 = coef_1 * advantages
per_token_loss2 = coef_2 * advantages
per_token_loss = -torch.min(per_token_loss1, per_token_loss2)
else:
raise ValueError(f"Unknown loss type: {self.loss_type}")
Expand Down Expand Up @@ -1883,8 +1885,8 @@ def masked_batch_mean(x):

if self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]:
# Compute the clipped probability ratios
is_low_clipped = (coef_1 < 1 - self.epsilon_low) & (advantages.unsqueeze(1) < 0)
is_high_clipped = (coef_1 > 1 + self.epsilon_high) & (advantages.unsqueeze(1) > 0)
is_low_clipped = (coef_1 < 1 - self.epsilon_low) & (advantages < 0)
is_high_clipped = (coef_1 > 1 + self.epsilon_high) & (advantages > 0)
is_region_clipped = is_low_clipped | is_high_clipped

low_clip = masked_batch_mean(is_low_clipped.float())
Expand All @@ -1900,7 +1902,7 @@ def masked_batch_mean(x):
gathered_clip_ratio = self.accelerator.gather(clip_ratio)
self._metrics[mode]["clip_ratio/region_mean"].append(gathered_clip_ratio.nanmean().item())
elif self.loss_type == "cispo":
is_cispo_clipped = (coef_1 > self.epsilon_high) & (advantages.unsqueeze(1) > 0)
is_cispo_clipped = (coef_1 > self.epsilon_high) & (advantages > 0)
cispo_clip_ratio = masked_batch_mean(is_cispo_clipped.float())
gathered_cispo_clip_ratio = self.accelerator.gather(cispo_clip_ratio)
self._metrics[mode]["cispo_clip_ratio"].append(gathered_cispo_clip_ratio.nanmean().item())
Expand Down
150 changes: 150 additions & 0 deletions trl/trainer/minillm_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
# Copyright 2020-2025 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import warnings
from dataclasses import dataclass, field
from typing import Any

from transformers import TrainingArguments
from .grpo_config import GRPOConfig


@dataclass
class MiniLLMConfig(GRPOConfig):
"""
Configuration class for [`MiniLLMTrainer`].

This class includes only the parameters that are specific to MiniLLM training. For a full list of training arguments,
please refer to the [`~transformers.TrainingArguments`] and [`SFTConfig`] documentation.

Args:
temperature (`float`, *optional*, defaults to `0.9`):
Temperature for sampling. The higher the temperature, the more random the completions.
lmbda (`float`, *optional*, defaults to `0.5`):
Lambda parameter that controls the student data fraction (i.e., the proportion of on-policy
student-generated outputs).
beta (`float`, *optional*, defaults to `0.5`):
Interpolation coefficient between `0.0` and `1.0` of the Generalized Jensen-Shannon Divergence loss. When
beta is `0.0`, the loss is the KL divergence. When beta is `1.0`, the loss is the Inverse KL Divergence.
max_new_tokens (`int`, *optional*, defaults to `128`):
Maximum number of tokens to generate per completion.
teacher_model_name_or_path (`str`, *optional*):
Model name or path of the teacher model. If `None`, the teacher model will be the same as the model being
trained.
teacher_model_init_kwargs (`dict[str, Any]]`, *optional*):
Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the teacher model
from a string.
disable_dropout (`bool`, *optional*, defaults to `True`):
Whether to disable dropout in the model.
seq_kd (`bool`, *optional*, defaults to `False`):
Seq_kd parameter that controls whether to perform Sequence-Level KD (can be viewed as supervised FT on
teacher-generated output).
"""
teacher_model_init_kwargs: dict[str, Any] | None = field(
default=None,
metadata={
"help": "Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the "
"teacher model from a string."
},
)
disable_dropout: bool = field(
default=True,
metadata={"help": "Whether to disable dropouts in `model`."},
)
single_step_decomposition: bool = field(
default=True,
metadata={
"help": "Whether to use single-step decomposition for the KL divergence computation."
},
)
kd_temperature: float = field(
default=1.0,
metadata={
"help": "Temperature for knowledge distillation. Higher temperatures produce softer probability "
"distributions over classes."
},
)
gamma: float = field(
default=0.0,
metadata={
"help": "Discount factor for future rewards in reinforcement learning."
},
)
length_normalization: bool = field(
default=True,
metadata={
"help": "Whether to apply length normalization to the rewards."
},
)

def __post_init__(self):
# We do not use the post_init of GRPOConfig because:
# 1. num_generations can be < 2 in MiniLLMConfig. Scale_rewards must be set to "none" to avoid nan.
self.bf16 = not (self.fp16) if self.bf16 is None else self.bf16

TrainingArguments.__post_init__(self)

self.scale_rewards = {True: "group", False: "none"}.get(self.scale_rewards, self.scale_rewards)
if self.num_generations == 1:
self.scale_rewards = "none"

num_processes = self.world_size
# The current default effective batch size
if self.generation_batch_size is None and self.steps_per_generation is None:
self.steps_per_generation = self.gradient_accumulation_steps
self.generation_batch_size = self.per_device_train_batch_size * num_processes * self.steps_per_generation
elif self.generation_batch_size is not None and self.steps_per_generation is None:
# Just ensure the value is divisible by the global batch size
if self.generation_batch_size % (self.per_device_train_batch_size * num_processes) != 0:
raise ValueError(
f"generation_batch_size ({self.generation_batch_size}) must be divisible by the global batch size "
f"({self.per_device_train_batch_size * num_processes})."
)
self.steps_per_generation = self.generation_batch_size // (
self.per_device_train_batch_size * num_processes
)
elif self.generation_batch_size is None and self.steps_per_generation is not None:
self.generation_batch_size = self.per_device_train_batch_size * num_processes * self.steps_per_generation
else:
raise ValueError(
"'generation_batch_size' and 'steps_per_generation' can not be both configured at the same time"
)

if self.do_eval and self.eval_strategy != "no":
# Just ensure the value is divisible by the global batch size
if (self.per_device_eval_batch_size * num_processes) % self.num_generations != 0:
raise ValueError(
f"The global eval batch size ({self.per_device_eval_batch_size} * {num_processes}) must be "
f"divisible by num_generations ({self.num_generations})."
)

# The generation batch must contain full prompt groups (no partials), so it must be divisible by
# num_generations.
if self.generation_batch_size % self.num_generations != 0:
raise ValueError(
f"generation_batch_size ({self.generation_batch_size}) must be divisible by num_generations "
f"({self.num_generations})."
)

if self.use_liger_loss is not None:
warnings.warn(
"The `use_liger_loss` argument is deprecated and will be removed in version 0.28.0. Please use "
"`use_liger_kernel` instead.",
FutureWarning,
stacklevel=2,
)
self.use_liger_kernel = self.use_liger_loss

if self.delta is not None and self.use_liger_kernel:
raise ValueError("Liger kernel does not support two-sided GRPO loss yet.")
Loading