diff --git a/README.md b/README.md index c0a49208262..9f11409a5f7 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,9 @@ trainer = GRPOTrainer( trainer.train() ``` +> [NOTE!] +> For reasoning models, use the `reasoning_accuracy_reward()` function for better results. + ### `DPOTrainer` [`DPOTrainer`](https://huggingface.co/docs/trl/dpo_trainer) implements the popular [Direct Preference Optimization (DPO) algorithm](https://huggingface.co/papers/2305.18290) that was used to post-train [Llama 3](https://huggingface.co/papers/2407.21783) and many other models. Here is a basic example of how to use the `DPOTrainer`: diff --git a/docs/source/lora_without_regret.md b/docs/source/lora_without_regret.md index f56e4ef9200..064c48f4be5 100644 --- a/docs/source/lora_without_regret.md +++ b/docs/source/lora_without_regret.md @@ -135,94 +135,6 @@ The blog post performs GRPO on a range of models and datasets from the Hub, and For reinforcement learning, the blog uses a math reasoning task that we can reproduce as a Python function. -
-Reward function - -```python -def strip_reasoning_accuracy_reward( - completions: list[list[dict[str, str]]], solution: list[str], **kwargs -) -> list[float | None]: - """Reward function that strips reasoning tags and checks mathematical accuracy. - - This function: - 1. Extracts the content from completions - 2. Removes tags (for reasoning that shouldn't be evaluated) - 3. Parses both the gold solution and the predicted answer - 4. Uses math_verify to check if they are mathematically equivalent - - Args: - completions: List of model completions, each containing a list of messages - solution: List of ground truth solutions - **kwargs: Additional arguments (ignored but required for trainer compatibility) - - Returns: - List of rewards where: - - 1.0 if the answer is correct - - 0.0 if the answer is incorrect - - None if the solution is not parseable (skips this example) - """ - contents = [completion[0]["content"] for completion in completions] - rewards = [] - - for content, sol in zip(contents, solution): - # Strip reasoning tags from completion - while "" in content and "" in content: - start = content.find("") - end = content.find("", start) - if start != -1 and end != -1: - content = content[:start] + content[end + len("") :] - else: - break - - # Parse gold solution - gold_parsed = parse( - f"${sol}$", - extraction_config=[ - LatexExtractionConfig( - boxed_match_priority=0, try_extract_without_anchor=True - ) - ], - ) - - if len(gold_parsed) != 0: - # We require the answer to be provided in correct latex (no malformed operators) - answer_parsed = parse( - content, - extraction_config=[ - LatexExtractionConfig( - boxed_match_priority=0, - normalization_config=NormalizationConfig( - basic_latex=True, - units=True, - malformed_operators=False, - nits=False, - boxed=True, - ), - try_extract_without_anchor=False, - ) - ], - extraction_mode="first_match", - ) - - # Compute binary rewards if verifiable, `None` otherwise to skip this example - try: - reward = float(verify(gold_parsed, answer_parsed)) - except Exception as e: - print( - f"verify failed: {e}, answer: {answer_parsed}, gold: {gold_parsed}" - ) - reward = None - else: - # If the gold solution is not parseable, we assign `None` to skip this example - reward = None - - rewards.append(reward) - - return rewards -``` - -
- @@ -233,14 +145,10 @@ We can implement these recommendations with the TRL Python API like so: from datasets import load_dataset from peft import LoraConfig from trl import GRPOConfig, GRPOTrainer +from trl.rewards import reasoning_accuracy_reward dataset = load_dataset("HuggingFaceH4/OpenR1-Math-220k-default-verified", split="train") -def strip_reasoning_accuracy_reward(completions, **kwargs): - """Reward function that strips reasoning and accuracy scores from the model outputs.""" - - ... - peft_config = LoraConfig( r=1, lora_alpha=32, @@ -259,7 +167,7 @@ training_args = GRPOConfig( trainer = GRPOTrainer( model="Qwen/Qwen3-0.6B", - reward_funcs=strip_reasoning_accuracy_reward, + reward_funcs=reasoning_accuracy_reward, args=training_args, train_dataset=dataset, peft_config=peft_config, diff --git a/docs/source/rewards.md b/docs/source/rewards.md index d18d6402f02..52752205f37 100644 --- a/docs/source/rewards.md +++ b/docs/source/rewards.md @@ -6,6 +6,10 @@ This module contains some useful reward functions, primarily intended for use wi [[autodoc]] rewards.accuracy_reward +## reasoning_accuracy_reward + +[[autodoc]] rewards.reasoning_accuracy_reward + ## think_format_reward [[autodoc]] rewards.think_format_reward diff --git a/tests/test_rewards.py b/tests/test_rewards.py index 8a8ce906146..f8e1f97b115 100644 --- a/tests/test_rewards.py +++ b/tests/test_rewards.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward +from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward from .testing_utils import TrlTestCase, require_math_latex @@ -117,27 +117,76 @@ def test_accuracy_reward_unparseable_gold(self): """Test accuracy_reward with an unparseable gold solution.""" completion = [ [{"content": "Answer is forty two."}], - [{"content": "Some other content."}], - [{"content": r"Answer is \boxed{42}."}], - [{"content": r"Answer is \boxed{\mathbf{42}}."}], # Make response bold - [{"content": r"Answer is \boxed{\textbf{42}}."}], # Different latex command for bold - [{"content": r"Answer is \boxed{42}."}], - [{"content": r"Answer is \boxed{42.3456}."}], + [{"content": r"Some other content. \boxed{43}."}], ] solution = [ "Answer is forty two.", "Answer is forty three.", - "Answer is 42.0", # Decimal point - "Answer is 42 43 okay?", # Extra space - "Answer is 42", - r"Answer is \n\boxed{42}", # Newline in gold solution - "Answer is 42.34560", # Extra trailing zero ] rewards = accuracy_reward(completion, solution) - assert rewards[0] == 1.0 # Should revert to exact text match + assert rewards[0] is None + assert rewards[1] is None + + +class TestReasoningAccuracyReward: + @require_math_latex + def test_correct_answer_yields_unit_reward(self): + completions = [ + [{"content": r" Reasoning content \boxed{\frac{63}{400}}"}], + [{"content": r"Reasoning content \boxed{\frac{63}{400}}"}], + ] + solutions = [r"\frac{63}{400}", r"\frac{63}{400}"] + rewards = reasoning_accuracy_reward(completions, solutions) + assert rewards[0] == 1.0 + assert rewards[1] == 1.0 + + @require_math_latex + def test_correct_answer_with_custom_tags_yields_unit_reward(self): + completions = [ + [{"content": r" Reasoning content \boxed{\frac{63}{400}}"}], + ] + solutions = [ + r"\frac{63}{400}", + ] + rewards = reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=[""]) + assert rewards[0] == 1.0 + + @require_math_latex + def test_incorrect_answer_yields_zero_reward(self): + completion = [[{"content": r" Reasoning content \boxed{\frac{64}{400}}"}]] + solution = [r"\frac{63}{400}"] + rewards = reasoning_accuracy_reward(completion, solution) + assert rewards[0] == 0.0 + + @require_math_latex + def test_correct_answer_in_reasoning_yields_zero_reward(self): + completions = [ + [{"content": r" My answer is \boxed{42} Some other text."}], + [{"content": r" The answer is \boxed{42} Here's a wrong answer: \boxed{43}."}], + ] + solutions = [r"\boxed{42}", r"\boxed{42}"] + rewards = reasoning_accuracy_reward(completions, solutions) + assert rewards[0] == 0.0 + assert rewards[1] == 0.0 + + @require_math_latex + def test_incomplete_reasoning_yields_zero_reward(self): + completions = [ + [{"content": r" Incomplete reasoning without closing tag"}], + [{"content": r"Correct answer \frac{63}{400} but completely missing reasoning content"}], + ] + solutions = [r"\frac{63}{400}", r"\frac{63}{400}"] + rewards = reasoning_accuracy_reward(completions, solutions) + assert rewards[0] == 0.0 assert rewards[1] == 0.0 - assert rewards[2] == 1.0 - assert rewards[3] == 1.0 - assert rewards[4] == 1.0 - assert rewards[5] == 1.0 - assert rewards[6] == 1.0 # Should ignore trailing zeros + + @require_math_latex + def test_unparseable_gold_solution_yields_none_reward(self): + completions = [ + [{"content": r" Reasoning content \boxed{42}"}], + ] + solutions = [ + "forty two", + ] + rewards = reasoning_accuracy_reward(completions, solutions) + assert rewards[0] is None diff --git a/trl/rewards/__init__.py b/trl/rewards/__init__.py index ce10ce99fe2..4d9cccff27a 100644 --- a/trl/rewards/__init__.py +++ b/trl/rewards/__init__.py @@ -19,14 +19,14 @@ _import_structure = { - "accuracy_rewards": ["accuracy_reward"], + "accuracy_rewards": ["accuracy_reward", "reasoning_accuracy_reward"], "format_rewards": ["think_format_reward"], "other_rewards": ["get_soft_overlong_punishment"], } if TYPE_CHECKING: - from .accuracy_rewards import accuracy_reward + from .accuracy_rewards import accuracy_reward, reasoning_accuracy_reward from .format_rewards import think_format_reward from .other_rewards import get_soft_overlong_punishment diff --git a/trl/rewards/accuracy_rewards.py b/trl/rewards/accuracy_rewards.py index cb02ee83a9c..6126158ecdc 100644 --- a/trl/rewards/accuracy_rewards.py +++ b/trl/rewards/accuracy_rewards.py @@ -22,9 +22,9 @@ def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str], **kwargs) -> list[float | None]: r""" - Reward function that checks if the completion is the same as the ground truth. + Reward function that checks if the completion matches the ground truth. - If both gold and prediction are parseable → use math verification. - - If not parseable → compare as normalized text. + - If gold is not parseable → return `None` to skip the example. Args: completions (`list[list[dict[str, str]]]`): @@ -39,12 +39,12 @@ def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str] ```python >>> from trl.rewards import accuracy_reward - >>> solution = [r"\frac{1}{3}", r"\frac{1}{3}"] - >>> completion = [ + >>> solutions = [r"\frac{1}{3}", r"\frac{1}{3}"] + >>> completions = [ ... [{"role": "assistant", "content": r"My answer is \boxed{\frac{1}{3}}"}], ... [{"role": "assistant", "content": r"My answer is \boxed{\frac{1}{2}}"}], ... ] - >>> accuracy_reward(completion, solution) + >>> accuracy_reward(completions, solutions) [1.0, 0.0] ``` """ @@ -69,11 +69,110 @@ def accuracy_reward(completions: list[list[dict[str, str]]], solution: list[str] ], extraction_mode="first_match", ) - # Compute binary rewards if verifiable, `None` otherwise to skip this example reward = float(verify(gold_parsed, answer_parsed)) else: - # If the gold solution is not parseable, we assign `None` to skip this example - reward = float(content.strip().lower() == sol.strip().lower()) + # If the gold solution cannot be parsed, we assign `None` to skip this example + reward = None + rewards.append(reward) + + return rewards + + +def reasoning_accuracy_reward( + completions: list[list[dict[str, str]]], + solution: list[str], + reasoning_delimiters: list[str] | None = None, + **kwargs, +) -> list[float | None]: + r""" + Reward function that removes the reasoning content and checks if the final answer matches the ground truth. + - If both gold and prediction are parseable → use math verification. + - If gold is not parseable → return `None` to skip the example. + + Args: + completions (`list[list[dict[str, str]]]`): + List of completions to be evaluated. Each completion must be a list of one message, i.e. a dictionary + containing the key `"content"` with the value being the text of the completion. + solution: (`list[str]`): + List of the raw-text solutions to the questions/problems/prompts. + reasoning_delimiters (`list[str]]`, *optional*): + List of strings indicating where the reasoning content ends. The final answer is assumed to be after the + last occurrence of any of these delimiters. If `None`, defaults to `[""]`. + **kwargs: + Additional keyword arguments. This function does not use them, but they are required in the function + signature to ensure compatibility with trainers like [`GRPOTrainer`]. + Example: + ```python + >>> from trl.rewards import reasoning_accuracy_reward + + >>> reasoning_delimiters = [""] + >>> solutions = [r"\frac{1}{3}", r"\frac{1}{3}", r"\frac{1}{3}"] + >>> completions = [ + ... [ + ... { + ... "role": "assistant", + ... "content": r" Reasoning content The final answer is \boxed{\frac{1}{3}}", + ... } + ... ], + ... [ + ... { + ... "role": "assistant", + ... "content": r" Reasoning content The final answer is \boxed{\frac{1}{2}}", + ... } + ... ], + ... [ + ... { + ... "role": "assistant", + ... "content": r" Reasoning content with partial answers \boxed{\frac{1}{3}} but no final answer", + ... } + ... ], + ... ] + >>> reasoning_accuracy_reward(completions, solutions, reasoning_delimiters=reasoning_delimiters) + [1.0, 0.0, 0.0] + ``` + """ + if not is_math_verify_available(): + raise ImportError("Please install the `math_verify` package to use reasoning_accuracy_reward") + + if reasoning_delimiters is None: + # Use sensible defaults for majority of reasoning models + reasoning_delimiters = [""] + + rewards = [] + contents = [completion[0]["content"] for completion in completions] + for content, sol in zip(contents, solution, strict=True): + # Split final answer from reasoning content + is_reasoning_complete = False + for delim in reasoning_delimiters: + if delim in content: + content = content.split(delim)[-1] + is_reasoning_complete = True + break + if not is_reasoning_complete: + # We assign zero reward instead of `None` to penalize incomplete reasoning + rewards.append(0.0) + continue + + gold_parsed = parse(sol) + if len(gold_parsed) != 0: + # We require the answer to be provided in correct latex (no malformed operators) + answer_parsed = parse( + content, + extraction_config=[ + LatexExtractionConfig( + boxed_match_priority=0, + normalization_config=NormalizationConfig( + units=True, + ), + try_extract_without_anchor=False, + ) + ], + extraction_mode="first_match", + ) + reward = float(verify(gold_parsed, answer_parsed)) + else: + # If the gold solution cannot be parsed, we assign `None` to skip this example + reward = None rewards.append(reward) return rewards diff --git a/trl/scripts/grpo.py b/trl/scripts/grpo.py index ece0ae55808..106623d4358 100644 --- a/trl/scripts/grpo.py +++ b/trl/scripts/grpo.py @@ -43,7 +43,7 @@ get_peft_config, get_quantization_config, ) -from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward +from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward logger = logging.get_logger(__name__) @@ -54,6 +54,7 @@ reward_funcs_registry = { "accuracy_reward": accuracy_reward, + "reasoning_accuracy_reward": reasoning_accuracy_reward, "think_format_reward": think_format_reward, "get_soft_overlong_punishment": get_soft_overlong_punishment(max_completion_len=1280, soft_punish_cache=256), } @@ -70,8 +71,8 @@ class GRPOScriptArguments(ScriptArguments): directory containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`]. reward_funcs (`list[str]`, *optional*): Reward functions to use. Supported values are: - - `"accuracy_reward"` + - `"reasoning_accuracy_reward"` - `"think_format_reward"` - `"get_soft_overlong_punishment"` (used value are `max_completion_len=1280`, `soft_punish_cache=256`) - any dotted import path " (e.g., `'my_lib.rewards.custom_reward'`). @@ -87,8 +88,8 @@ class GRPOScriptArguments(ScriptArguments): reward_funcs: list[str] | None = field( default=None, metadata={ - "help": "Reward functions to use. Supported values are: `accuracy_reward`, `think_format_reward`, " - "`get_soft_overlong_punishment` (used value are `max_completion_len=1280`, `soft_punish_cache=256`), or " + "help": "Reward functions to use. Supported values are: `accuracy_reward`, `reasoning_accuracy_reward`, `think_format_reward`, " + "`get_soft_overlong_punishment` (used values are `max_completion_len=1280`, `soft_punish_cache=256`), or " "any dotted import path (e.g., `'my_lib.rewards.custom_reward'`)." }, ) diff --git a/trl/scripts/rloo.py b/trl/scripts/rloo.py index 9545d0f11c6..d5a87aaddc3 100644 --- a/trl/scripts/rloo.py +++ b/trl/scripts/rloo.py @@ -40,7 +40,7 @@ get_dataset, get_peft_config, ) -from trl.rewards import accuracy_reward, get_soft_overlong_punishment, think_format_reward +from trl.rewards import accuracy_reward, get_soft_overlong_punishment, reasoning_accuracy_reward, think_format_reward logger = logging.get_logger(__name__) @@ -51,6 +51,7 @@ reward_funcs_registry = { "accuracy_reward": accuracy_reward, + "reasoning_accuracy_reward": reasoning_accuracy_reward, "think_format_reward": think_format_reward, "get_soft_overlong_punishment": get_soft_overlong_punishment(max_completion_len=1280, soft_punish_cache=256), } @@ -67,8 +68,8 @@ class RLOOScriptArguments(ScriptArguments): directory containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`]. reward_funcs (`list[str]`, *optional*): Reward functions to use. Supported values are: - - `"accuracy_reward"` + - `"reasoning_accuracy_reward"` - `"think_format_reward"` - `"get_soft_overlong_punishment"` (used value are `max_completion_len=1280`, `soft_punish_cache=256`) - any dotted import path " (e.g., `'my_lib.rewards.custom_reward'`). @@ -84,8 +85,8 @@ class RLOOScriptArguments(ScriptArguments): reward_funcs: list[str] | None = field( default=None, metadata={ - "help": "Reward functions to use. Supported values are: `accuracy_reward`, `think_format_reward`, " - "`get_soft_overlong_punishment` (used value are `max_completion_len=1280`, `soft_punish_cache=256`), or " + "help": "Reward functions to use. Supported values are: `accuracy_reward`, `reasoning_accuracy_reward`, `think_format_reward`, " + "`get_soft_overlong_punishment` (used values are `max_completion_len=1280`, `soft_punish_cache=256`), or " "any dotted import path (e.g., `'my_lib.rewards.custom_reward'`)." }, )