Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ trainer = GRPOTrainer(
trainer.train()
```

> [NOTE!]
> For reasoning models, use the `reasoning_accuracy_reward()` function for better results.

### `DPOTrainer`

[`DPOTrainer`](https://huggingface.co/docs/trl/dpo_trainer) implements the popular [Direct Preference Optimization (DPO) algorithm](https://huggingface.co/papers/2305.18290) that was used to post-train [Llama 3](https://huggingface.co/papers/2407.21783) and many other models. Here is a basic example of how to use the `DPOTrainer`:
Expand Down
96 changes: 2 additions & 94 deletions docs/source/lora_without_regret.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,94 +135,6 @@ The blog post performs GRPO on a range of models and datasets from the Hub, and

For reinforcement learning, the blog uses a math reasoning task that we can reproduce as a Python function.

<details>
<summary>Reward function</summary>

```python
def strip_reasoning_accuracy_reward(

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For viz @burtenshaw a close variant of this reward function is now native in TRL as reasoning_accuracy_reward()

completions: list[list[dict[str, str]]], solution: list[str], **kwargs
) -> list[float | None]:
"""Reward function that strips reasoning tags and checks mathematical accuracy.

This function:
1. Extracts the content from completions
2. Removes <think></think> tags (for reasoning that shouldn't be evaluated)
3. Parses both the gold solution and the predicted answer
4. Uses math_verify to check if they are mathematically equivalent

Args:
completions: List of model completions, each containing a list of messages
solution: List of ground truth solutions
**kwargs: Additional arguments (ignored but required for trainer compatibility)

Returns:
List of rewards where:
- 1.0 if the answer is correct
- 0.0 if the answer is incorrect
- None if the solution is not parseable (skips this example)
"""
contents = [completion[0]["content"] for completion in completions]
rewards = []

for content, sol in zip(contents, solution):
# Strip reasoning tags from completion
while "<think>" in content and "</think>" in content:
start = content.find("<think>")
end = content.find("</think>", start)
if start != -1 and end != -1:
content = content[:start] + content[end + len("</think>") :]
else:
break

# Parse gold solution
gold_parsed = parse(
f"${sol}$",
extraction_config=[
LatexExtractionConfig(
boxed_match_priority=0, try_extract_without_anchor=True
)
],
)

if len(gold_parsed) != 0:
# We require the answer to be provided in correct latex (no malformed operators)
answer_parsed = parse(
content,
extraction_config=[
LatexExtractionConfig(
boxed_match_priority=0,
normalization_config=NormalizationConfig(
basic_latex=True,
units=True,
malformed_operators=False,
nits=False,
boxed=True,
),
try_extract_without_anchor=False,
)
],
extraction_mode="first_match",
)

# Compute binary rewards if verifiable, `None` otherwise to skip this example
try:
reward = float(verify(gold_parsed, answer_parsed))
except Exception as e:
print(
f"verify failed: {e}, answer: {answer_parsed}, gold: {gold_parsed}"
)
reward = None
else:
# If the gold solution is not parseable, we assign `None` to skip this example
reward = None

rewards.append(reward)

return rewards
```

</details>

<hfoptions id="grpo">
<hfoption id="python">

Expand All @@ -233,14 +145,10 @@ We can implement these recommendations with the TRL Python API like so:
from datasets import load_dataset
from peft import LoraConfig
from trl import GRPOConfig, GRPOTrainer
from trl.rewards import reasoning_accuracy_reward

dataset = load_dataset("HuggingFaceH4/OpenR1-Math-220k-default-verified", split="train")

def strip_reasoning_accuracy_reward(completions, **kwargs):
"""Reward function that strips reasoning and accuracy scores from the model outputs."""

...

peft_config = LoraConfig(
r=1,
lora_alpha=32,
Expand All @@ -259,7 +167,7 @@ training_args = GRPOConfig(

trainer = GRPOTrainer(
model="Qwen/Qwen3-0.6B",
reward_funcs=strip_reasoning_accuracy_reward,
reward_funcs=reasoning_accuracy_reward,
args=training_args,
train_dataset=dataset,
peft_config=peft_config,
Expand Down
4 changes: 4 additions & 0 deletions docs/source/rewards.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ This module contains some useful reward functions, primarily intended for use wi

[[autodoc]] rewards.accuracy_reward

## reasoning_accuracy_reward

[[autodoc]] rewards.reasoning_accuracy_reward

## think_format_reward

[[autodoc]] rewards.think_format_reward
Expand Down
85 changes: 67 additions & 18 deletions tests/test_rewards.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward
from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward

from .testing_utils import TrlTestCase, require_math_latex

Expand Down Expand Up @@ -117,27 +117,76 @@ def test_accuracy_reward_unparseable_gold(self):
"""Test accuracy_reward with an unparseable gold solution."""
completion = [
[{"content": "Answer is forty two."}],
[{"content": "Some other content."}],

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed these cases as they weren't testing what was advertised :)

[{"content": r"Answer is \boxed{42}."}],
[{"content": r"Answer is \boxed{\mathbf{42}}."}], # Make response bold
[{"content": r"Answer is \boxed{\textbf{42}}."}], # Different latex command for bold
[{"content": r"Answer is \boxed{42}."}],
[{"content": r"Answer is \boxed{42.3456}."}],
[{"content": r"Some other content. \boxed{43}."}],
]
solution = [
"Answer is forty two.",
"Answer is forty three.",
"Answer is 42.0", # Decimal point
"Answer is 42 43 okay?", # Extra space
"Answer is 42",
r"Answer is \n\boxed{42}", # Newline in gold solution
"Answer is 42.34560", # Extra trailing zero
]
rewards = accuracy_reward(completion, solution)
assert rewards[0] == 1.0 # Should revert to exact text match
assert rewards[0] is None
assert rewards[1] is None


class TestReasoningAccuracyReward:
@require_math_latex
def test_correct_answer_yields_unit_reward(self):
completions = [
[{"content": r"<think> Reasoning content </think> \boxed{\frac{63}{400}}"}],
[{"content": r"Reasoning content </think> \boxed{\frac{63}{400}}"}],
]
solutions = [r"\frac{63}{400}", r"\frac{63}{400}"]
rewards = reasoning_accuracy_reward(completions, solutions)
assert rewards[0] == 1.0
assert rewards[1] == 1.0

@require_math_latex
def test_correct_answer_with_custom_tags_yields_unit_reward(self):
completions = [
[{"content": r"<REASONING_START> Reasoning content </REASONING_END> \boxed{\frac{63}{400}}"}],
]
solutions = [
r"\frac{63}{400}",
]
rewards = reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=["</REASONING_END>"])
assert rewards[0] == 1.0

@require_math_latex
def test_incorrect_answer_yields_zero_reward(self):
completion = [[{"content": r"<think> Reasoning content </think> \boxed{\frac{64}{400}}"}]]
solution = [r"\frac{63}{400}"]
rewards = reasoning_accuracy_reward(completion, solution)
assert rewards[0] == 0.0

@require_math_latex
def test_correct_answer_in_reasoning_yields_zero_reward(self):
completions = [
[{"content": r"<think> My answer is \boxed{42} </think> Some other text."}],
[{"content": r"<think> The answer is \boxed{42} </think> Here's a wrong answer: \boxed{43}."}],
]
solutions = [r"\boxed{42}", r"\boxed{42}"]
rewards = reasoning_accuracy_reward(completions, solutions)
assert rewards[0] == 0.0
assert rewards[1] == 0.0

@require_math_latex
def test_incomplete_reasoning_yields_zero_reward(self):
completions = [
[{"content": r"<think> Incomplete reasoning without closing tag"}],
[{"content": r"Correct answer \frac{63}{400} but completely missing reasoning content"}],
]
solutions = [r"\frac{63}{400}", r"\frac{63}{400}"]
rewards = reasoning_accuracy_reward(completions, solutions)
assert rewards[0] == 0.0
assert rewards[1] == 0.0
assert rewards[2] == 1.0
assert rewards[3] == 1.0
assert rewards[4] == 1.0
assert rewards[5] == 1.0
assert rewards[6] == 1.0 # Should ignore trailing zeros

@require_math_latex
def test_unparseable_gold_solution_yields_none_reward(self):
completions = [
[{"content": r"<think> Reasoning content </think> \boxed{42}"}],
]
solutions = [
"forty two",
]
rewards = reasoning_accuracy_reward(completions, solutions)
assert rewards[0] is None
4 changes: 2 additions & 2 deletions trl/rewards/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@


_import_structure = {
"accuracy_rewards": ["accuracy_reward"],
"accuracy_rewards": ["accuracy_reward", "reasoning_accuracy_reward"],
"format_rewards": ["think_format_reward"],
"other_rewards": ["get_soft_overlong_punishment"],
}


if TYPE_CHECKING:
from .accuracy_rewards import accuracy_reward
from .accuracy_rewards import accuracy_reward, reasoning_accuracy_reward
from .format_rewards import think_format_reward
from .other_rewards import get_soft_overlong_punishment

Expand Down
Loading