diff --git a/README.md b/README.md
index c0a49208262..9f11409a5f7 100644
--- a/README.md
+++ b/README.md
@@ -104,6 +104,9 @@ trainer = GRPOTrainer(
trainer.train()
```
+> [NOTE!]
+> For reasoning models, use the `reasoning_accuracy_reward()` function for better results.
+
### `DPOTrainer`
[`DPOTrainer`](https://huggingface.co/docs/trl/dpo_trainer) implements the popular [Direct Preference Optimization (DPO) algorithm](https://huggingface.co/papers/2305.18290) that was used to post-train [Llama 3](https://huggingface.co/papers/2407.21783) and many other models. Here is a basic example of how to use the `DPOTrainer`:
diff --git a/docs/source/lora_without_regret.md b/docs/source/lora_without_regret.md
index f56e4ef9200..064c48f4be5 100644
--- a/docs/source/lora_without_regret.md
+++ b/docs/source/lora_without_regret.md
@@ -135,94 +135,6 @@ The blog post performs GRPO on a range of models and datasets from the Hub, and
For reinforcement learning, the blog uses a math reasoning task that we can reproduce as a Python function.
-
-Reward function
-
-```python
-def strip_reasoning_accuracy_reward(
- completions: list[list[dict[str, str]]], solution: list[str], **kwargs
-) -> list[float | None]:
- """Reward function that strips reasoning tags and checks mathematical accuracy.
-
- This function:
- 1. Extracts the content from completions
- 2. Removes tags (for reasoning that shouldn't be evaluated)
- 3. Parses both the gold solution and the predicted answer
- 4. Uses math_verify to check if they are mathematically equivalent
-
- Args:
- completions: List of model completions, each containing a list of messages
- solution: List of ground truth solutions
- **kwargs: Additional arguments (ignored but required for trainer compatibility)
-
- Returns:
- List of rewards where:
- - 1.0 if the answer is correct
- - 0.0 if the answer is incorrect
- - None if the solution is not parseable (skips this example)
- """
- contents = [completion[0]["content"] for completion in completions]
- rewards = []
-
- for content, sol in zip(contents, solution):
- # Strip reasoning tags from completion
- while "" in content and "" in content:
- start = content.find("")
- end = content.find("", start)
- if start != -1 and end != -1:
- content = content[:start] + content[end + len("") :]
- else:
- break
-
- # Parse gold solution
- gold_parsed = parse(
- f"${sol}$",
- extraction_config=[
- LatexExtractionConfig(
- boxed_match_priority=0, try_extract_without_anchor=True
- )
- ],
- )
-
- if len(gold_parsed) != 0:
- # We require the answer to be provided in correct latex (no malformed operators)
- answer_parsed = parse(
- content,
- extraction_config=[
- LatexExtractionConfig(
- boxed_match_priority=0,
- normalization_config=NormalizationConfig(
- basic_latex=True,
- units=True,
- malformed_operators=False,
- nits=False,
- boxed=True,
- ),
- try_extract_without_anchor=False,
- )
- ],
- extraction_mode="first_match",
- )
-
- # Compute binary rewards if verifiable, `None` otherwise to skip this example
- try:
- reward = float(verify(gold_parsed, answer_parsed))
- except Exception as e:
- print(
- f"verify failed: {e}, answer: {answer_parsed}, gold: {gold_parsed}"
- )
- reward = None
- else:
- # If the gold solution is not parseable, we assign `None` to skip this example
- reward = None
-
- rewards.append(reward)
-
- return rewards
-```
-
-
-
@@ -233,14 +145,10 @@ We can implement these recommendations with the TRL Python API like so:
from datasets import load_dataset
from peft import LoraConfig
from trl import GRPOConfig, GRPOTrainer
+from trl.rewards import reasoning_accuracy_reward
dataset = load_dataset("HuggingFaceH4/OpenR1-Math-220k-default-verified", split="train")
-def strip_reasoning_accuracy_reward(completions, **kwargs):
- """Reward function that strips reasoning and accuracy scores from the model outputs."""
-
- ...
-
peft_config = LoraConfig(
r=1,
lora_alpha=32,
@@ -259,7 +167,7 @@ training_args = GRPOConfig(
trainer = GRPOTrainer(
model="Qwen/Qwen3-0.6B",
- reward_funcs=strip_reasoning_accuracy_reward,
+ reward_funcs=reasoning_accuracy_reward,
args=training_args,
train_dataset=dataset,
peft_config=peft_config,
diff --git a/docs/source/rewards.md b/docs/source/rewards.md
index d18d6402f02..52752205f37 100644
--- a/docs/source/rewards.md
+++ b/docs/source/rewards.md
@@ -6,6 +6,10 @@ This module contains some useful reward functions, primarily intended for use wi
[[autodoc]] rewards.accuracy_reward
+## reasoning_accuracy_reward
+
+[[autodoc]] rewards.reasoning_accuracy_reward
+
## think_format_reward
[[autodoc]] rewards.think_format_reward
diff --git a/tests/test_rewards.py b/tests/test_rewards.py
index 8a8ce906146..f8e1f97b115 100644
--- a/tests/test_rewards.py
+++ b/tests/test_rewards.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward
+from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward
from .testing_utils import TrlTestCase, require_math_latex
@@ -117,27 +117,76 @@ def test_accuracy_reward_unparseable_gold(self):
"""Test accuracy_reward with an unparseable gold solution."""
completion = [
[{"content": "Answer is forty two."}],
- [{"content": "Some other content."}],
- [{"content": r"Answer is \boxed{42}."}],
- [{"content": r"Answer is \boxed{\mathbf{42}}."}], # Make response bold
- [{"content": r"Answer is \boxed{\textbf{42}}."}], # Different latex command for bold
- [{"content": r"Answer is \boxed{42}."}],
- [{"content": r"Answer is \boxed{42.3456}."}],
+ [{"content": r"Some other content. \boxed{43}."}],
]
solution = [
"Answer is forty two.",
"Answer is forty three.",
- "Answer is 42.0", # Decimal point
- "Answer is 42 43 okay?", # Extra space
- "Answer is 42",
- r"Answer is \n\boxed{42}", # Newline in gold solution
- "Answer is 42.34560", # Extra trailing zero
]
rewards = accuracy_reward(completion, solution)
- assert rewards[0] == 1.0 # Should revert to exact text match
+ assert rewards[0] is None
+ assert rewards[1] is None
+
+
+class TestReasoningAccuracyReward:
+ @require_math_latex
+ def test_correct_answer_yields_unit_reward(self):
+ completions = [
+ [{"content": r" Reasoning content \boxed{\frac{63}{400}}"}],
+ [{"content": r"Reasoning content \boxed{\frac{63}{400}}"}],
+ ]
+ solutions = [r"\frac{63}{400}", r"\frac{63}{400}"]
+ rewards = reasoning_accuracy_reward(completions, solutions)
+ assert rewards[0] == 1.0
+ assert rewards[1] == 1.0
+
+ @require_math_latex
+ def test_correct_answer_with_custom_tags_yields_unit_reward(self):
+ completions = [
+ [{"content": r" Reasoning content \boxed{\frac{63}{400}}"}],
+ ]
+ solutions = [
+ r"\frac{63}{400}",
+ ]
+ rewards = reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=[""])
+ assert rewards[0] == 1.0
+
+ @require_math_latex
+ def test_incorrect_answer_yields_zero_reward(self):
+ completion = [[{"content": r" Reasoning content \boxed{\frac{64}{400}}"}]]
+ solution = [r"\frac{63}{400}"]
+ rewards = reasoning_accuracy_reward(completion, solution)
+ assert rewards[0] == 0.0
+
+ @require_math_latex
+ def test_correct_answer_in_reasoning_yields_zero_reward(self):
+ completions = [
+ [{"content": r" My answer is \boxed{42} Some other text."}],
+ [{"content": r" The answer is \boxed{42} Here's a wrong answer: \boxed{43}."}],
+ ]
+ solutions = [r"\boxed{42}", r"\boxed{42}"]
+ rewards = reasoning_accuracy_reward(completions, solutions)
+ assert rewards[0] == 0.0
+ assert rewards[1] == 0.0
+
+ @require_math_latex
+ def test_incomplete_reasoning_yields_zero_reward(self):
+ completions = [
+ [{"content": r" Incomplete reasoning without closing tag"}],
+ [{"content": r"Correct answer \frac{63}{400} but completely missing reasoning content"}],
+ ]
+ solutions = [r"\frac{63}{400}", r"\frac{63}{400}"]
+ rewards = reasoning_accuracy_reward(completions, solutions)
+ assert rewards[0] == 0.0
assert rewards[1] == 0.0
- assert rewards[2] == 1.0
- assert rewards[3] == 1.0
- assert rewards[4] == 1.0
- assert rewards[5] == 1.0
- assert rewards[6] == 1.0 # Should ignore trailing zeros
+
+ @require_math_latex
+ def test_unparseable_gold_solution_yields_none_reward(self):
+ completions = [
+ [{"content": r" Reasoning content \boxed{42}"}],
+ ]
+ solutions = [
+ "forty two",
+ ]
+ rewards = reasoning_accuracy_reward(completions, solutions)
+ assert rewards[0] is None
diff --git a/trl/rewards/__init__.py b/trl/rewards/__init__.py
index ce10ce99fe2..4d9cccff27a 100644
--- a/trl/rewards/__init__.py
+++ b/trl/rewards/__init__.py
@@ -19,14 +19,14 @@
_import_structure = {
- "accuracy_rewards": ["accuracy_reward"],
+ "accuracy_rewards": ["accuracy_reward", "reasoning_accuracy_reward"],
"format_rewards": ["think_format_reward"],
"other_rewards": ["get_soft_overlong_punishment"],
}
if TYPE_CHECKING:
- from .accuracy_rewards import accuracy_reward
+ from .accuracy_rewards import accuracy_reward, reasoning_accuracy_reward
from .format_rewards import think_format_reward
from .other_rewards import get_soft_overlong_punishment
diff --git a/trl/rewards/accuracy_rewards.py b/trl/rewards/accuracy_rewards.py
index cb02ee83a9c..6126158ecdc 100644
--- a/trl/rewards/accuracy_rewards.py
+++ b/trl/rewards/accuracy_rewards.py
@@ -22,9 +22,9 @@
def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str], **kwargs) -> list[float | None]:
r"""
- Reward function that checks if the completion is the same as the ground truth.
+ Reward function that checks if the completion matches the ground truth.
- If both gold and prediction are parseable → use math verification.
- - If not parseable → compare as normalized text.
+ - If gold is not parseable → return `None` to skip the example.
Args:
completions (`list[list[dict[str, str]]]`):
@@ -39,12 +39,12 @@ def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str]
```python
>>> from trl.rewards import accuracy_reward
- >>> solution = [r"\frac{1}{3}", r"\frac{1}{3}"]
- >>> completion = [
+ >>> solutions = [r"\frac{1}{3}", r"\frac{1}{3}"]
+ >>> completions = [
... [{"role": "assistant", "content": r"My answer is \boxed{\frac{1}{3}}"}],
... [{"role": "assistant", "content": r"My answer is \boxed{\frac{1}{2}}"}],
... ]
- >>> accuracy_reward(completion, solution)
+ >>> accuracy_reward(completions, solutions)
[1.0, 0.0]
```
"""
@@ -69,11 +69,110 @@ def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str]
],
extraction_mode="first_match",
)
- # Compute binary rewards if verifiable, `None` otherwise to skip this example
reward = float(verify(gold_parsed, answer_parsed))
else:
- # If the gold solution is not parseable, we assign `None` to skip this example
- reward = float(content.strip().lower() == sol.strip().lower())
+ # If the gold solution cannot be parsed, we assign `None` to skip this example
+ reward = None
+ rewards.append(reward)
+
+ return rewards
+
+
+def reasoning_accuracy_reward(
+ completions: list[list[dict[str, str]]],
+ solution: list[str],
+ reasoning_delimiters: list[str] | None = None,
+ **kwargs,
+) -> list[float | None]:
+ r"""
+ Reward function that removes the reasoning content and checks if the final answer matches the ground truth.
+ - If both gold and prediction are parseable → use math verification.
+ - If gold is not parseable → return `None` to skip the example.
+
+ Args:
+ completions (`list[list[dict[str, str]]]`):
+ List of completions to be evaluated. Each completion must be a list of one message, i.e. a dictionary
+ containing the key `"content"` with the value being the text of the completion.
+ solution: (`list[str]`):
+ List of the raw-text solutions to the questions/problems/prompts.
+ reasoning_delimiters (`list[str]]`, *optional*):
+ List of strings indicating where the reasoning content ends. The final answer is assumed to be after the
+ last occurrence of any of these delimiters. If `None`, defaults to `[""]`.
+ **kwargs:
+ Additional keyword arguments. This function does not use them, but they are required in the function
+ signature to ensure compatibility with trainers like [`GRPOTrainer`].
+ Example:
+ ```python
+ >>> from trl.rewards import reasoning_accuracy_reward
+
+ >>> reasoning_delimiters = [""]
+ >>> solutions = [r"\frac{1}{3}", r"\frac{1}{3}", r"\frac{1}{3}"]
+ >>> completions = [
+ ... [
+ ... {
+ ... "role": "assistant",
+ ... "content": r" Reasoning content The final answer is \boxed{\frac{1}{3}}",
+ ... }
+ ... ],
+ ... [
+ ... {
+ ... "role": "assistant",
+ ... "content": r" Reasoning content The final answer is \boxed{\frac{1}{2}}",
+ ... }
+ ... ],
+ ... [
+ ... {
+ ... "role": "assistant",
+ ... "content": r" Reasoning content with partial answers \boxed{\frac{1}{3}} but no final answer",
+ ... }
+ ... ],
+ ... ]
+ >>> reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=reasoning_delimiters)
+ [1.0, 0.0, 0.0]
+ ```
+ """
+ if not is_math_verify_available():
+ raise ImportError("Please install the `math_verify` package to use reasoning_accuracy_reward")
+
+ if reasoning_delimiters is None:
+ # Use sensible defaults for majority of reasoning models
+ reasoning_delimiters = [""]
+
+ rewards = []
+ contents = [completion[0]["content"] for completion in completions]
+ for content, sol in zip(contents, solution, strict=True):
+ # Split final answer from reasoning content
+ is_reasoning_complete = False
+ for delim in reasoning_delimiters:
+ if delim in content:
+ content = content.split(delim)[-1]
+ is_reasoning_complete = True
+ break
+ if not is_reasoning_complete:
+ # We assign zero reward instead of `None` to penalize incomplete reasoning
+ rewards.append(0.0)
+ continue
+
+ gold_parsed = parse(sol)
+ if len(gold_parsed) != 0:
+ # We require the answer to be provided in correct latex (no malformed operators)
+ answer_parsed = parse(
+ content,
+ extraction_config=[
+ LatexExtractionConfig(
+ boxed_match_priority=0,
+ normalization_config=NormalizationConfig(
+ units=True,
+ ),
+ try_extract_without_anchor=False,
+ )
+ ],
+ extraction_mode="first_match",
+ )
+ reward = float(verify(gold_parsed, answer_parsed))
+ else:
+ # If the gold solution cannot be parsed, we assign `None` to skip this example
+ reward = None
rewards.append(reward)
return rewards
diff --git a/trl/scripts/grpo.py b/trl/scripts/grpo.py
index ece0ae55808..106623d4358 100644
--- a/trl/scripts/grpo.py
+++ b/trl/scripts/grpo.py
@@ -43,7 +43,7 @@
get_peft_config,
get_quantization_config,
)
-from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward
+from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward
logger = logging.get_logger(__name__)
@@ -54,6 +54,7 @@
reward_funcs_registry = {
"accuracy_reward": accuracy_reward,
+ "reasoning_accuracy_reward": reasoning_accuracy_reward,
"think_format_reward": think_format_reward,
"get_soft_overlong_punishment": get_soft_overlong_punishment(max_completion_len=1280, soft_punish_cache=256),
}
@@ -70,8 +71,8 @@ class GRPOScriptArguments(ScriptArguments):
directory containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`].
reward_funcs (`list[str]`, *optional*):
Reward functions to use. Supported values are:
-
- `"accuracy_reward"`
+ - `"reasoning_accuracy_reward"`
- `"think_format_reward"`
- `"get_soft_overlong_punishment"` (used value are `max_completion_len=1280`, `soft_punish_cache=256`)
- any dotted import path " (e.g., `'my_lib.rewards.custom_reward'`).
@@ -87,8 +88,8 @@ class GRPOScriptArguments(ScriptArguments):
reward_funcs: list[str] | None = field(
default=None,
metadata={
- "help": "Reward functions to use. Supported values are: `accuracy_reward`, `think_format_reward`, "
- "`get_soft_overlong_punishment` (used value are `max_completion_len=1280`, `soft_punish_cache=256`), or "
+ "help": "Reward functions to use. Supported values are: `accuracy_reward`, `reasoning_accuracy_reward`, `think_format_reward`, "
+ "`get_soft_overlong_punishment` (used values are `max_completion_len=1280`, `soft_punish_cache=256`), or "
"any dotted import path (e.g., `'my_lib.rewards.custom_reward'`)."
},
)
diff --git a/trl/scripts/rloo.py b/trl/scripts/rloo.py
index 9545d0f11c6..d5a87aaddc3 100644
--- a/trl/scripts/rloo.py
+++ b/trl/scripts/rloo.py
@@ -40,7 +40,7 @@
get_dataset,
get_peft_config,
)
-from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward
+from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward
logger = logging.get_logger(__name__)
@@ -51,6 +51,7 @@
reward_funcs_registry = {
"accuracy_reward": accuracy_reward,
+ "reasoning_accuracy_reward": reasoning_accuracy_reward,
"think_format_reward": think_format_reward,
"get_soft_overlong_punishment": get_soft_overlong_punishment(max_completion_len=1280, soft_punish_cache=256),
}
@@ -67,8 +68,8 @@ class RLOOScriptArguments(ScriptArguments):
directory containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`].
reward_funcs (`list[str]`, *optional*):
Reward functions to use. Supported values are:
-
- `"accuracy_reward"`
+ - `"reasoning_accuracy_reward"`
- `"think_format_reward"`
- `"get_soft_overlong_punishment"` (used value are `max_completion_len=1280`, `soft_punish_cache=256`)
- any dotted import path " (e.g., `'my_lib.rewards.custom_reward'`).
@@ -84,8 +85,8 @@ class RLOOScriptArguments(ScriptArguments):
reward_funcs: list[str] | None = field(
default=None,
metadata={
- "help": "Reward functions to use. Supported values are: `accuracy_reward`, `think_format_reward`, "
- "`get_soft_overlong_punishment` (used value are `max_completion_len=1280`, `soft_punish_cache=256`), or "
+ "help": "Reward functions to use. Supported values are: `accuracy_reward`, `reasoning_accuracy_reward`, `think_format_reward`, "
+ "`get_soft_overlong_punishment` (used values are `max_completion_len=1280`, `soft_punish_cache=256`), or "
"any dotted import path (e.g., `'my_lib.rewards.custom_reward'`)."
},
)