From 7bbfc962e4627e62d1f740e42500de580c6d1839 Mon Sep 17 00:00:00 2001 From: Pramodith Ballapuram <16939722+pramodith@users.noreply.github.com> Date: Thu, 6 Nov 2025 23:37:22 +0000 Subject: [PATCH 1/2] Fix bugs! --- trl/trainer/grpo_trainer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 74a85563d49..e40136e4b2a 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -1824,10 +1824,9 @@ def _compute_loss(self, model, inputs): # From here, log_importance_weights (and all subsequent tensors, coef_1, coef_2, etc.) shape depends on # importance_sampling_level: "token" level: (B, T); "sequence" level: (B, 1) - if self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]: + if self.loss_type == "cispo": clamped_ratios = torch.clamp(coef_1, max=self.epsilon_high).detach() per_token_loss = -clamped_ratios * advantages.unsqueeze(1) * per_token_logps - else: coef_2 = torch.clamp(coef_1, 1 - self.epsilon_low, 1 + self.epsilon_high) # Two-sided clipping @@ -1880,7 +1879,7 @@ def masked_batch_mean(x): mean_entropy = masked_batch_mean(entropies) self._metrics[mode]["entropy"].append(self.accelerator.gather(mean_entropy).nanmean().item()) - if self.loss_type != "cispo": + if self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]: # Compute the clipped probability ratios is_low_clipped = (coef_1 < 1 - self.epsilon_low) & (advantages.unsqueeze(1) < 0) is_high_clipped = (coef_1 > 1 + self.epsilon_high) & (advantages.unsqueeze(1) > 0) From f21570191b7890652bf2b5f58b9af05c9ee34cbf Mon Sep 17 00:00:00 2001 From: Pramodith Ballapuram <16939722+pramodith@users.noreply.github.com> Date: Thu, 6 Nov 2025 23:52:49 +0000 Subject: [PATCH 2/2] nits. --- trl/trainer/grpo_trainer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index e40136e4b2a..8e8021635ec 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -1827,7 +1827,7 @@ def _compute_loss(self, model, inputs): if self.loss_type == "cispo": clamped_ratios = torch.clamp(coef_1, max=self.epsilon_high).detach() per_token_loss = -clamped_ratios * advantages.unsqueeze(1) * per_token_logps - else: + elif self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]: coef_2 = torch.clamp(coef_1, 1 - self.epsilon_low, 1 + self.epsilon_high) # Two-sided clipping if self.args.delta is not None: @@ -1836,6 +1836,8 @@ def _compute_loss(self, model, inputs): per_token_loss1 = coef_1 * advantages.unsqueeze(1) per_token_loss2 = coef_2 * advantages.unsqueeze(1) per_token_loss = -torch.min(per_token_loss1, per_token_loss2) + else: + raise ValueError(f"Unknown loss type: {self.loss_type}") if entropy_mask is not None: per_token_loss = per_token_loss * entropy_mask