From 8fd7109c1b719a727f0e91fd2b732ec596b85bc9 Mon Sep 17 00:00:00 2001 From: Behrooz Date: Wed, 5 Nov 2025 13:52:59 -0800 Subject: [PATCH 1/8] refactor: Move CPOTrainer to experimental module Move CPOTrainer and CPOConfig to trl.experimental.cpo following the pattern established in #4312 for BCO. Changes: - Create trl/experimental/cpo/ module with CPOTrainer and CPOConfig - Replace original files with deprecation stubs - Update all imports in tests, examples, and documentation - Move CPO from Trainers to Experimental section in docs - Maintain backward compatibility with deprecation warnings until TRL 0.29 Closes #4460 --- docs/source/_toctree.yml | 4 +- docs/source/cpo_trainer.md | 2 +- docs/source/paper_index.md | 2 +- examples/scripts/cpo.py | 3 +- tests/test_cpo_trainer.py | 2 +- tests/test_trainers_args.py | 3 +- trl/experimental/cpo/__init__.py | 16 + trl/experimental/cpo/cpo_config.py | 218 ++++++ trl/experimental/cpo/cpo_trainer.py | 1090 +++++++++++++++++++++++++++ trl/trainer/cpo_config.py | 207 +---- trl/trainer/cpo_trainer.py | 1012 +------------------------ 11 files changed, 1361 insertions(+), 1198 deletions(-) create mode 100644 trl/experimental/cpo/__init__.py create mode 100644 trl/experimental/cpo/cpo_config.py create mode 100644 trl/experimental/cpo/cpo_trainer.py diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 74f79544b33..533dfa6ca80 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -56,8 +56,6 @@ title: Examples - sections: - sections: # Sorted alphabetically - - local: cpo_trainer - title: CPO - local: dpo_trainer title: DPO - local: online_dpo_trainer @@ -109,6 +107,8 @@ title: BEMA for Reference Model - local: bco_trainer title: BCO + - local: cpo_trainer + title: CPO - local: gfpo title: GFPO - local: gold_trainer diff --git a/docs/source/cpo_trainer.md b/docs/source/cpo_trainer.md index 3dcdb0e11cd..c2c0f7746b1 100644 --- a/docs/source/cpo_trainer.md +++ b/docs/source/cpo_trainer.md @@ -24,7 +24,7 @@ Below is the script to train the model: ```python # train_cpo.py from datasets import load_dataset -from trl import CPOConfig, CPOTrainer +from trl.experimental.cpo import CPOConfig, CPOTrainer from transformers import AutoModelForCausalLM, AutoTokenizer model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") diff --git a/docs/source/paper_index.md b/docs/source/paper_index.md index 6467548d8ea..11292f4fa96 100644 --- a/docs/source/paper_index.md +++ b/docs/source/paper_index.md @@ -543,7 +543,7 @@ Papers relating to the [`CPOTrainer`] AlphaPO is a new Direct Alignment Algorithms (DAAs) method that leverages an alpha-parameter to help change the shape of the reward function beyond the standard log reward. AlphaPO helps maintain fine-grained control over likelihood displacement and over-optimization. To reproduce the paper's setting, use this configuration: ```python -from trl import CPOConfig +from trl.experimental.cpo import CPOConfig # Mistral-Instruct from Table 3 of the paper training_args = CPOConfig( diff --git a/examples/scripts/cpo.py b/examples/scripts/cpo.py index fef9cdf1247..5d621d9eae1 100644 --- a/examples/scripts/cpo.py +++ b/examples/scripts/cpo.py @@ -63,7 +63,8 @@ from datasets import load_dataset from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser -from trl import CPOConfig, CPOTrainer, ModelConfig, ScriptArguments, get_peft_config +from trl import ModelConfig, ScriptArguments, get_peft_config +from trl.experimental.cpo import CPOConfig, CPOTrainer # Enable logging in a Hugging Face Space diff --git a/tests/test_cpo_trainer.py b/tests/test_cpo_trainer.py index a346277c07f..7aecc54df5d 100644 --- a/tests/test_cpo_trainer.py +++ b/tests/test_cpo_trainer.py @@ -17,7 +17,7 @@ from datasets import load_dataset from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer -from trl import CPOConfig, CPOTrainer +from trl.experimental.cpo import CPOConfig, CPOTrainer from .testing_utils import TrlTestCase, require_peft diff --git a/tests/test_trainers_args.py b/tests/test_trainers_args.py index 014ec6ac5da..f49efc65c78 100644 --- a/tests/test_trainers_args.py +++ b/tests/test_trainers_args.py @@ -17,8 +17,6 @@ from transformers import AutoModelForCausalLM, AutoModelForSequenceClassification, AutoTokenizer from trl import ( - CPOConfig, - CPOTrainer, DPOConfig, DPOTrainer, FDivergenceType, @@ -37,6 +35,7 @@ XPOConfig, XPOTrainer, ) +from trl.experimental.cpo import CPOConfig, CPOTrainer from .testing_utils import TrlTestCase diff --git a/trl/experimental/cpo/__init__.py b/trl/experimental/cpo/__init__.py new file mode 100644 index 00000000000..d542f227fba --- /dev/null +++ b/trl/experimental/cpo/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020-2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cpo_config import CPOConfig +from .cpo_trainer import CPOTrainer diff --git a/trl/experimental/cpo/cpo_config.py b/trl/experimental/cpo/cpo_config.py new file mode 100644 index 00000000000..426b00da153 --- /dev/null +++ b/trl/experimental/cpo/cpo_config.py @@ -0,0 +1,218 @@ +# Copyright 2020-2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any + +from transformers import TrainingArguments + + +@dataclass +class CPOConfig(TrainingArguments): + r""" + Configuration class for the [`CPOTrainer`]. + + This class includes only the parameters that are specific to CPO training. For a full list of training arguments, + please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may + differ from those in [`~transformers.TrainingArguments`]. + + Using [`~transformers.HfArgumentParser`] we can turn this class into + [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the + command line. + + Parameters: + max_length (`int` or `None`, *optional*, defaults to `1024`): + Maximum length of the sequences (prompt + completion) in the batch. This argument is required if you want + to use the default data collator. + max_prompt_length (`int` or `None`, *optional*, defaults to `512`): + Maximum length of the prompt. This argument is required if you want to use the default data collator. + max_completion_length (`int`, *optional*): + Maximum length of the completion. This argument is required if you want to use the default data collator + and your model is an encoder-decoder. + beta (`float`, *optional*, defaults to `0.1`): + Parameter controlling the deviation from the reference model. Higher β means less deviation from the + reference model. For the IPO loss (`loss_type="ipo"`), β is the regularization parameter denoted by τ in + the [paper](https://huggingface.co/papers/2310.12036). + label_smoothing (`float`, *optional*, defaults to `0.0`): + Label smoothing factor. This argument is required if you want to use the default data collator. + loss_type (`str`, *optional*, defaults to `"sigmoid"`): + Type of loss to use. Possible values are: + + - `"sigmoid"`: sigmoid loss from the original [DPO](https://huggingface.co/papers/2305.18290) paper. + - `"hinge"`: hinge loss on the normalized likelihood from the + [SLiC](https://huggingface.co/papers/2305.10425) paper. + - `"ipo"`: IPO loss from the [IPO](https://huggingface.co/papers/2310.12036) paper. + - `"simpo"`: SimPO loss from the [SimPO](https://huggingface.co/papers/2405.14734) paper. + - `"alphapo"`: AlphaPO loss from the [AlphaPO](https://huggingface.co/papers/2501.03884) paper. This + automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`. + + disable_dropout (`bool`, *optional*, defaults to `True`): + Whether to disable dropout in the model. + cpo_alpha (`float`, *optional*, defaults to `1.0`): + Weight of the BC regularizer in CPO training. + simpo_gamma (`float`, *optional*, defaults to `0.5`): + Target reward margin for the SimPO loss, used only when the `loss_type="simpo"`. + alpha (`float`, *optional*, defaults to `0.0`): + Alpha parameter that controls reward function shape across all loss types. When alpha=0 (default), uses + standard log probability rewards. When `alpha != 0`, applies AlphaPO transformation: `r = (1 - p^(-alpha)) + / alpha` from the [AlphaPO paper](https://huggingface.co/papers/2501.03884). This parameter works with all + loss types. + label_pad_token_id (`int`, *optional*, defaults to `-100`): + Label pad token id. This argument is required if you want to use the default data collator. + padding_value (`int`, *optional*): + Padding value to use. If `None`, the padding value of the tokenizer is used. + truncation_mode (`str`,*optional*, defaults to `"keep_end"`): + Truncation mode to use when the prompt is too long. Possible values are `"keep_end"` or `"keep_start"`. + This argument is required if you want to use the default data collator. + generate_during_eval (`bool`, *optional*, defaults to `False`): + If `True`, generates and logs completions from the model to W&B or Comet during evaluation. + is_encoder_decoder (`bool`, *optional*): + When using the `model_init` argument (callable) to instantiate the model instead of the `model` argument, + you need to specify if the model returned by the callable is an encoder-decoder model. + model_init_kwargs (`dict[str, Any]`, *optional*): + Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the model from a + string. + dataset_num_proc (`int`, *optional*): + Number of processes to use for processing the dataset. + """ + + _VALID_DICT_FIELDS = TrainingArguments._VALID_DICT_FIELDS + ["model_init_kwargs"] + + # Parameters whose default values are overridden from TrainingArguments + learning_rate: float = field( + default=1e-6, + metadata={"help": "The initial learning rate for AdamW."}, + ) + logging_steps: float = field( + default=10, + metadata={ + "help": "Log every X updates steps. Should be an integer or a float in range `[0,1)`. If smaller than 1, " + "will be interpreted as ratio of total training steps." + }, + ) + gradient_checkpointing: bool = field( + default=True, + metadata={ + "help": "If True, use gradient checkpointing to save memory at the expense of slower backward pass." + }, + ) + bf16: bool | None = field( + default=None, + metadata={ + "help": "Whether to use bf16 (mixed) precision instead of 32-bit. Requires Ampere or higher NVIDIA " + "architecture or Intel XPU or using CPU (use_cpu) or Ascend NPU. If not set, it defaults to `True` if " + "`fp16` is not set." + }, + ) + + max_length: int | None = field( + default=1024, + metadata={"help": "Maximum length of the sequences (prompt + completion) in the batch."}, + ) + max_prompt_length: int | None = field( + default=512, + metadata={ + "help": "Maximum length of the prompt. This argument is required if you want to use the default data " + "collator and your model is an encoder-decoder." + }, + ) + max_completion_length: int | None = field( + default=None, + metadata={ + "help": "Maximum length of the completion. This argument is required if you want to use the default data " + "collator and your model is an encoder-decoder." + }, + ) + beta: float = field( + default=0.1, + metadata={ + "help": "Parameter controlling the deviation from the reference model. Higher β means less deviation from " + "the reference model." + }, + ) + label_smoothing: float = field( + default=0.0, + metadata={"help": "Label smoothing factor."}, + ) + loss_type: str = field( + default="sigmoid", + metadata={ + "help": "Type of loss to use.", + "choices": ["sigmoid", "hinge", "ipo", "simpo", "alphapo"], + }, + ) + disable_dropout: bool = field( + default=True, + metadata={"help": "Whether to disable dropout in the model."}, + ) + cpo_alpha: float = field( + default=1.0, + metadata={"help": "Weight of the BC regularizer in CPO training."}, + ) + simpo_gamma: float = field( + default=0.5, + metadata={"help": "Target reward margin for the SimPO loss, used only when the `loss_type='simpo'`."}, + ) + alpha: float = field( + default=0.0, + metadata={ + "help": "Alpha parameter that controls reward function shape across all loss types. When alpha=0 " + "(default), uses standard log probability rewards. When `alpha != 0`, applies AlphaPO transformation: " + "`r = (1 - p^(-alpha)) / alpha` from the AlphaPO paper. This parameter works with all loss types." + }, + ) + label_pad_token_id: int = field( + default=-100, + metadata={"help": "Label pad token id."}, + ) + padding_value: int | None = field( + default=None, + metadata={"help": "Padding value to use. If `None`, the padding value of the tokenizer is used."}, + ) + truncation_mode: str = field( + default="keep_end", + metadata={ + "help": "Truncation mode to use when the prompt is too long.", + "choices": ["keep_end", "keep_start"], + }, + ) + generate_during_eval: bool = field( + default=False, + metadata={"help": "If `True`, generates and logs completions from the model to W&B during evaluation."}, + ) + is_encoder_decoder: bool | None = field( + default=None, + metadata={"help": "Whether the model is an encoder-decoder model."}, + ) + model_init_kwargs: dict[str, Any] | None = field( + default=None, + metadata={ + "help": "Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the model " + "from a string." + }, + ) + dataset_num_proc: int | None = field( + default=None, + metadata={"help": "Number of processes to use for processing the dataset."}, + ) + + def __post_init__(self): + self.bf16 = not (self.fp16) if self.bf16 is None else self.bf16 + + # Syntactic sugar for AlphaPO: set loss_type to "simpo" and cpo_alpha to 0.0 + if self.loss_type == "alphapo": + self.loss_type = "simpo" + self.cpo_alpha = 0.0 + + super().__post_init__() diff --git a/trl/experimental/cpo/cpo_trainer.py b/trl/experimental/cpo/cpo_trainer.py new file mode 100644 index 00000000000..30974f60888 --- /dev/null +++ b/trl/experimental/cpo/cpo_trainer.py @@ -0,0 +1,1090 @@ +# Copyright 2020-2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os +import random +import textwrap +import warnings +from collections import defaultdict +from collections.abc import Callable +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Literal + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +import torch.nn.functional as F +from accelerate import PartialState, logging +from datasets import Dataset +from torch import autocast +from torch.utils.data import DataLoader +from transformers import ( + AutoModelForCausalLM, + BaseImageProcessor, + DataCollator, + FeatureExtractionMixin, + PreTrainedModel, + PreTrainedTokenizerBase, + ProcessorMixin, + is_comet_available, + is_wandb_available, +) +from transformers.trainer_callback import TrainerCallback +from transformers.trainer_utils import EvalLoopOutput +from transformers.utils import is_peft_available, is_torch_fx_proxy + +from ...data_utils import maybe_apply_chat_template, maybe_extract_prompt +from ...trainer.base_trainer import BaseTrainer +from .cpo_config import CPOConfig +from ...trainer.utils import ( + DPODataCollatorWithPadding, + add_bos_token_if_needed, + add_eos_token_if_needed, + disable_dropout_in_model, + log_table_to_comet_experiment, + pad_to_length, + peft_module_casting_to_bf16, + selective_log_softmax, +) + + +if is_peft_available(): + from peft import PeftModel, get_peft_model, prepare_model_for_kbit_training + + +if is_wandb_available(): + import wandb + + +logger = logging.get_logger(__name__) + + +class CPOTrainer(BaseTrainer): + r""" + Initialize CPOTrainer. + + Args: + model ([`~transformers.PreTrainedModel`]): + The model to train, preferably an [`~transformers.AutoModelForSequenceClassification`]. + args ([`CPOConfig`]): + The CPO config arguments to use for training. + data_collator ([`~transformers.DataCollator`]): + The data collator to use for training. If None is specified, the default data collator + ([`DPODataCollatorWithPadding`]) will be used which will pad the sequences to the maximum length of the + sequences in the batch, given a dataset of paired sequences. + train_dataset ([`~datasets.Dataset`]): + The dataset to use for training. + eval_dataset ([`~datasets.Dataset`]): + The dataset to use for evaluation. + processing_class ([`~transformers.PreTrainedTokenizerBase`], [`~transformers.BaseImageProcessor`], [`~transformers.FeatureExtractionMixin`] or [`~transformers.ProcessorMixin`], *optional*): + Processing class used to process the data. If provided, will be used to automatically process the inputs + for the model, and it will be saved along the model to make it easier to rerun an interrupted training or + reuse the fine-tuned model. + model_init (`Callable[[], transformers.PreTrainedModel]`): + The model initializer to use for training. If None is specified, the default model initializer will be + used. + callbacks (`list[transformers.TrainerCallback]`): + The callbacks to use for training. + optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`): + The optimizer and scheduler to use for training. + preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`): + The function to use to preprocess the logits before computing the metrics. + peft_config (`dict`, defaults to `None`): + The PEFT configuration to use for training. If you pass a PEFT configuration, the model will be wrapped in + a PEFT model. + compute_metrics (`Callable[[EvalPrediction], dict]`, *optional*): + The function to use to compute the metrics. Must take a `EvalPrediction` and return a dictionary string to + metric values. + """ + + _tag_names = ["trl", "cpo"] + _name = "CPO" + _paper = { + "title": "Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation", + "id": "2401.08417", + # docstyle-ignore + "citation": textwrap.dedent("""\ + @inproceedings{xu2024contrastive, + title = {{Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation}}, + author = {Haoran Xu and Amr Sharaf and Yunmo Chen and Weiting Tan and Lingfeng Shen and Benjamin Van Durme and Kenton Murray and Young Jin Kim}, + year = 2024, + booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}, + publisher = {OpenReview.net}, + url = {https://openreview.net/forum?id=51iwkioZpn} + }"""), + } + + def __init__( + self, + model: PreTrainedModel | nn.Module | str | None = None, + args: CPOConfig | None = None, + data_collator: DataCollator | None = None, + train_dataset: Dataset | None = None, + eval_dataset: Dataset | dict[str, Dataset] | None = None, + processing_class: PreTrainedTokenizerBase + | BaseImageProcessor + | FeatureExtractionMixin + | ProcessorMixin + | None = None, + model_init: Callable[[], PreTrainedModel] | None = None, + callbacks: list[TrainerCallback] | None = None, + optimizers: tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), + preprocess_logits_for_metrics: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] | None = None, + peft_config: dict | None = None, + compute_metrics: Callable[[EvalLoopOutput], dict] | None = None, + ): + if args.model_init_kwargs is None: + model_init_kwargs = {} + elif not isinstance(model, str): + raise ValueError("You passed model_kwargs to the CPOTrainer. But your model is already instantiated.") + else: + model_init_kwargs = args.model_init_kwargs + dtype = model_init_kwargs.get("dtype") + if dtype is not None: + # Convert to `torch.dtype` if an str is passed + if isinstance(dtype, str) and dtype != "auto": + dtype = getattr(torch, dtype) + if dtype != "auto" and not isinstance(dtype, torch.dtype): + raise ValueError( + f"Invalid `dtype` passed to the CPOConfig. Expected a string with either `torch.dtype` or 'auto', but got {dtype}." + ) + model_init_kwargs["dtype"] = dtype + + if isinstance(model, str): + model = AutoModelForCausalLM.from_pretrained(model, **model_init_kwargs) + + # Initialize this variable to False. This helps tracking the case when `peft_module_casting_to_bf16` + # has been called in order to properly call autocast if needed. + self._peft_has_been_casted_to_bf16 = False + + if not is_peft_available() and peft_config is not None: + raise ValueError( + "PEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT models" + ) + elif is_peft_available() and peft_config is not None: + # if model is a peft model and we have a peft_config, we merge and unload it first + if isinstance(model, PeftModel): + model = model.merge_and_unload() + + if getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_loaded_in_4bit", False): + _support_gc_kwargs = hasattr( + args, "gradient_checkpointing_kwargs" + ) and "gradient_checkpointing_kwargs" in list( + inspect.signature(prepare_model_for_kbit_training).parameters + ) + + prepare_model_kwargs = {"use_gradient_checkpointing": args.gradient_checkpointing} + + if _support_gc_kwargs: + prepare_model_kwargs["gradient_checkpointing_kwargs"] = args.gradient_checkpointing_kwargs + + model = prepare_model_for_kbit_training(model, **prepare_model_kwargs) + elif args.gradient_checkpointing: + # For backward compatibility with older versions of transformers + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + # get peft model with the given config + model = get_peft_model(model, peft_config) + if args.bf16 and getattr(model, "is_loaded_in_4bit", False): + peft_module_casting_to_bf16(model) + # If args.bf16 we need to explicitly call `generate` with torch amp autocast context manager + self._peft_has_been_casted_to_bf16 = True + + # For models that use gradient_checkpointing, we need to attach a hook that enables input + # to explicitly have `requires_grad=True`, otherwise training will either silently + # fail or completely fail. + elif args.gradient_checkpointing: + # For backward compatibility with older versions of transformers + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + if args.generate_during_eval and not (is_wandb_available() or is_comet_available()): + raise ValueError( + "`generate_during_eval=True` requires Weights and Biases or Comet to be installed." + " Please install `wandb` or `comet-ml` to resolve." + ) + + if model is not None: + self.is_encoder_decoder = model.config.is_encoder_decoder + elif args.is_encoder_decoder is None: + raise ValueError("When no model is provided, you need to pass the parameter is_encoder_decoder.") + else: + self.is_encoder_decoder = args.is_encoder_decoder + + if self.is_encoder_decoder: + self.decoder_start_token_id = model.config.decoder_start_token_id + self.pad_token_id = model.config.pad_token_id + + if processing_class is None: + raise ValueError("processing_class must be specified to tokenize a CPO dataset.") + if args.max_length is None: + logger.warning( + "`max_length` is not set in the CPOConfig's init" + " it will default to `512` by default, but you should do it yourself in the future.", + ) + max_length = 512 + else: + max_length = args.max_length + if args.max_prompt_length is None: + logger.warning( + "`max_prompt_length` is not set in the CPOConfig's init" + " it will default to `128` by default, but you should do it yourself in the future.", + ) + max_prompt_length = 128 + else: + max_prompt_length = args.max_prompt_length + + if not max_prompt_length < max_length: + raise ValueError( + f"max_prompt_length ({max_prompt_length}) should be strictly less than max_length ({max_length})." + ) + + if args.max_completion_length is None and self.is_encoder_decoder: + logger.warning( + "When using an encoder decoder architecture, you should set `max_completion_length` in the CPOConfig's init" + " it will default to `128` by default, but you should do it yourself in the future.", + ) + max_completion_length = 128 + else: + max_completion_length = args.max_completion_length + + if data_collator is None: + data_collator = DPODataCollatorWithPadding( + pad_token_id=processing_class.pad_token_id, + label_pad_token_id=args.label_pad_token_id, + is_encoder_decoder=self.is_encoder_decoder, + ) + + if args.remove_unused_columns: + args.remove_unused_columns = False + # warn users + logger.warning( + "When using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments" + " we have set it for you, but you should do it yourself in the future.", + ) + + self.use_dpo_data_collator = True + else: + self.use_dpo_data_collator = False + + # Disable dropout in the model + if args.disable_dropout: + disable_dropout_in_model(model) + + self.max_length = max_length + self.generate_during_eval = args.generate_during_eval + self.label_pad_token_id = args.label_pad_token_id + self.padding_value = args.padding_value if args.padding_value is not None else processing_class.pad_token_id + self.max_prompt_length = max_prompt_length + self.truncation_mode = args.truncation_mode + self.max_completion_length = max_completion_length + self.processing_class = processing_class + + if args.loss_type in ["hinge", "ipo"] and args.label_smoothing > 0: + logger.warning( + f"You are using the {args.loss_type} loss type that does not support label smoothing. The " + "`label_smoothing` parameter will be ignored. Set `label_smoothing` to `0.0` to remove this warning.", + ) + if args.loss_type == "kto_pair": + raise ValueError("Support for kto_pair has been removed in CPOTrainer. Please use KTOTrainer.") + + self.beta = args.beta + self.label_smoothing = args.label_smoothing + self.loss_type = args.loss_type + self.cpo_alpha = args.cpo_alpha + self.aux_loss_enabled = getattr(model.config, "output_router_logits", False) + self.aux_loss_coef = getattr(model.config, "router_aux_loss_coef", 0.0) + if self.aux_loss_enabled and self.aux_loss_coef == 0.0: + logger.warning( + "You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to " + "`0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value " + "greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary " + "loss.", + ) + + if args.loss_type == "simpo": + self.simpo_gamma = args.simpo_gamma + + # AlphaPO parameter for reward shaping + self.alpha = args.alpha + + self._stored_metrics = defaultdict(lambda: defaultdict(list)) + + # The trainer estimates the number of FLOPs (floating-point operations) using the number of elements in the + # input tensor associated with the key "input_ids". However, in CPO, the sampled data does not include the + # "input_ids" key. Instead, the available keys are "prompt_input_ids", "chosen_input_ids", and + # "rejected_input_ids". As a result, the trainer issues the warning: "Could not estimate the number of tokens + # of the input, floating-point operations will not be computed." To suppress this warning, we set the + # "estimate_tokens" key in the model's "warnings_issued" dictionary to True. This acts as a flag to indicate + # that the warning has already been issued. + model.warnings_issued["estimate_tokens"] = True + + # Compute that only on the main process for faster data processing. + # see: https://github.com/huggingface/trl/pull/1255 + with PartialState().main_process_first(): + # Extract the prompt if needed, and apply the chat template if needed + train_dataset = train_dataset.map(maybe_extract_prompt, num_proc=args.dataset_num_proc) + train_dataset = train_dataset.map( + maybe_apply_chat_template, fn_kwargs={"tokenizer": processing_class}, num_proc=args.dataset_num_proc + ) + if eval_dataset is not None: + eval_dataset = eval_dataset.map(maybe_extract_prompt, num_proc=args.dataset_num_proc) + eval_dataset = eval_dataset.map( + maybe_apply_chat_template, + fn_kwargs={"tokenizer": processing_class}, + num_proc=args.dataset_num_proc, + ) + + # tokenize the dataset + train_dataset = train_dataset.map(self.tokenize_row, num_proc=args.dataset_num_proc) + if eval_dataset is not None: + eval_dataset = eval_dataset.map(self.tokenize_row, num_proc=args.dataset_num_proc) + + super().__init__( + model=model, + args=args, + data_collator=data_collator, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + processing_class=processing_class, + model_init=model_init, + compute_metrics=compute_metrics, + callbacks=callbacks, + optimizers=optimizers, + preprocess_logits_for_metrics=preprocess_logits_for_metrics, + ) + + # Gradient accumulation requires scaled loss. Normally, loss scaling in the parent class depends on whether the + # model accepts loss-related kwargs. Since we compute our own loss, this check is irrelevant. We set + # self.model_accepts_loss_kwargs to False to enable scaling. + self.model_accepts_loss_kwargs = False + + # Add tags for models that have been loaded with the correct transformers version + if hasattr(self.model, "add_model_tags"): + self.model.add_model_tags(self._tag_names) + + if not hasattr(self, "accelerator"): + raise AttributeError( + "Your `Trainer` does not have an `accelerator` object. Consider upgrading `transformers`." + ) + + def build_tokenized_answer(self, prompt, answer): + """ + Llama tokenizer does satisfy `enc(a + b) = enc(a) + enc(b)`. It does ensure `enc(a + b) = enc(a) + enc(a + + b)[len(enc(a)):]`. Reference: + https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 + """ + + full_tokenized = self.processing_class(prompt + answer, add_special_tokens=False) + prompt_input_ids = self.processing_class(prompt, add_special_tokens=False)["input_ids"] + + answer_input_ids = full_tokenized["input_ids"][len(prompt_input_ids) :] + answer_attention_mask = full_tokenized["attention_mask"][len(prompt_input_ids) :] + + # Concat tokens to form `enc(a) + enc(a + b)[len(enc(a)):]` + full_concat_input_ids = np.concatenate([prompt_input_ids, answer_input_ids]) + + # Prepare input tokens for token by token comparison + full_input_ids = np.array(full_tokenized["input_ids"]) + + if len(full_input_ids) != len(full_concat_input_ids): + raise ValueError("Prompt input ids and answer input ids should have the same length.") + + # On some tokenizers, like Llama-2 tokenizer, there are occasions where tokens + # can be merged together when tokenizing prompt+answer. This could result + # on the last token from the prompt being different when tokenized on its own + # vs when done as prompt+answer. + response_token_ids_start_idx = len(prompt_input_ids) + + # If tokenized prompt is different than both prompt+answer, then it means the + # last token has changed due to merging. + if prompt_input_ids != full_tokenized["input_ids"][:response_token_ids_start_idx]: + response_token_ids_start_idx -= 1 + + prompt_input_ids = full_tokenized["input_ids"][:response_token_ids_start_idx] + prompt_attention_mask = full_tokenized["attention_mask"][:response_token_ids_start_idx] + + if len(prompt_input_ids) != len(prompt_attention_mask): + raise ValueError("Prompt input ids and attention mask should have the same length.") + + answer_input_ids = full_tokenized["input_ids"][response_token_ids_start_idx:] + answer_attention_mask = full_tokenized["attention_mask"][response_token_ids_start_idx:] + + return dict( + prompt_input_ids=prompt_input_ids, + prompt_attention_mask=prompt_attention_mask, + input_ids=answer_input_ids, + attention_mask=answer_attention_mask, + ) + + def tokenize_row(self, feature, model: PreTrainedModel | nn.Module | None = None) -> dict: + """Tokenize a single row from a CPO specific dataset. + + At this stage, we don't convert to PyTorch tensors yet; we just handle the truncation in case the prompt + + chosen or prompt + rejected responses is/are too long. First we truncate the prompt; if we're still too long, + we truncate the chosen/rejected. + + We also create the labels for the chosen/rejected responses, which are of length equal to the sum of the length + of the prompt and the chosen/rejected response, with label_pad_token_id for the prompt tokens. + """ + batch = {} + prompt = feature["prompt"] + chosen = feature["chosen"] + rejected = feature["rejected"] + + if not self.is_encoder_decoder: + # Check issues below for more details + # 1. https://github.com/huggingface/trl/issues/907 + # 2. https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 + # 3. https://github.com/LianjiaTech/BELLE/issues/337 + + if not isinstance(prompt, str): + raise ValueError(f"prompt should be an str but got {type(prompt)}") + prompt_tokens = self.processing_class(prompt, add_special_tokens=False) + prompt_tokens = {f"prompt_{k}": v for k, v in prompt_tokens.items()} + + if not isinstance(chosen, str): + raise ValueError(f"chosen should be an str but got {type(chosen)}") + chosen_tokens = self.build_tokenized_answer(prompt, chosen) + + if not isinstance(rejected, str): + raise ValueError(f"rejected should be an str but got {type(rejected)}") + rejected_tokens = self.build_tokenized_answer(prompt, rejected) + + # Last prompt token might get merged by tokenizer and + # it should not be included for generation if that happens + prompt_len_input_ids = len(prompt_tokens["prompt_input_ids"]) + + chosen_prompt_len_input_ids = len(chosen_tokens["prompt_input_ids"]) + rejected_prompt_len_input_ids = len(rejected_tokens["prompt_input_ids"]) + prompt_len_input_ids = min(chosen_prompt_len_input_ids, rejected_prompt_len_input_ids) + + for k, v in prompt_tokens.items(): + prompt_tokens[k] = v[:prompt_len_input_ids] + + # Make sure prompts only have one different token at most an + # and length only differs by 1 at most + num_diff_tokens = sum( + a != b + for a, b in zip(chosen_tokens["prompt_input_ids"], rejected_tokens["prompt_input_ids"], strict=True) + ) + num_diff_len = abs(chosen_prompt_len_input_ids - rejected_prompt_len_input_ids) + if num_diff_tokens > 1 or num_diff_len > 1: + raise ValueError( + "Chosen and rejected prompt_input_ids might only differ on the " + "last token due to tokenizer merge ops." + ) + + # add BOS token to head of prompt. Avoid adding if it's already there + prompt_tokens, chosen_tokens, rejected_tokens = add_bos_token_if_needed( + self.processing_class.bos_token_id, + prompt_len_input_ids, + prompt_tokens, + chosen_prompt_len_input_ids, + chosen_tokens, + rejected_prompt_len_input_ids, + rejected_tokens, + ) + + # add EOS token to end of answer. Avoid adding if it's already there + chosen_tokens, rejected_tokens = add_eos_token_if_needed( + self.processing_class.eos_token_id, chosen_tokens, rejected_tokens + ) + + longer_response_length = max(len(chosen_tokens["input_ids"]), len(rejected_tokens["input_ids"])) + + # if combined sequence is too long, truncate the prompt + for answer_tokens in [chosen_tokens, rejected_tokens, prompt_tokens]: + if len(answer_tokens["prompt_input_ids"]) + longer_response_length > self.max_length: + if self.truncation_mode == "keep_start": + for k in ["prompt_input_ids", "prompt_attention_mask"]: + answer_tokens[k] = answer_tokens[k][: self.max_prompt_length] + elif self.truncation_mode == "keep_end": + for k in ["prompt_input_ids", "prompt_attention_mask"]: + answer_tokens[k] = answer_tokens[k][-self.max_prompt_length :] + else: + raise ValueError(f"Unknown truncation mode: {self.truncation_mode}") + + # if that's still too long, truncate the response + for answer_tokens in [chosen_tokens, rejected_tokens]: + if len(answer_tokens["prompt_input_ids"]) + longer_response_length > self.max_length: + for k in ["input_ids", "attention_mask"]: + answer_tokens[k] = answer_tokens[k][: self.max_length - self.max_prompt_length] + + # Create labels + chosen_sequence_tokens = { + k: chosen_tokens[f"prompt_{k}"] + chosen_tokens[k] for k in ["input_ids", "attention_mask"] + } + rejected_sequence_tokens = { + k: rejected_tokens[f"prompt_{k}"] + rejected_tokens[k] for k in ["input_ids", "attention_mask"] + } + chosen_sequence_tokens["labels"] = chosen_sequence_tokens["input_ids"][:] + chosen_sequence_tokens["labels"][: len(chosen_tokens["prompt_input_ids"])] = [ + self.label_pad_token_id + ] * len(chosen_tokens["prompt_input_ids"]) + rejected_sequence_tokens["labels"] = rejected_sequence_tokens["input_ids"][:] + rejected_sequence_tokens["labels"][: len(rejected_tokens["prompt_input_ids"])] = [ + self.label_pad_token_id + ] * len(rejected_tokens["prompt_input_ids"]) + + for k, toks in { + "chosen_": chosen_sequence_tokens, + "rejected_": rejected_sequence_tokens, + "": prompt_tokens, + }.items(): + for type_key, tokens in toks.items(): + if type_key == "token_type_ids": + continue + batch[f"{k}{type_key}"] = tokens + + else: + chosen_tokens = self.processing_class( + chosen, truncation=True, max_length=self.max_completion_length, add_special_tokens=True + ) + rejected_tokens = self.processing_class( + rejected, truncation=True, max_length=self.max_completion_length, add_special_tokens=True + ) + prompt_tokens = self.processing_class( + prompt, truncation=True, max_length=self.max_prompt_length, add_special_tokens=True + ) + + batch["chosen_labels"] = chosen_tokens["input_ids"] + batch["rejected_labels"] = rejected_tokens["input_ids"] + batch["prompt_input_ids"] = prompt_tokens["input_ids"] + batch["prompt_attention_mask"] = prompt_tokens["attention_mask"] + + if model is not None and hasattr(model, "prepare_decoder_input_ids_from_labels"): + batch["rejected_decoder_input_ids"] = model.prepare_decoder_input_ids_from_labels( + labels=torch.tensor(batch["rejected_labels"]) + ) + batch["chosen_decoder_input_ids"] = model.prepare_decoder_input_ids_from_labels( + labels=torch.tensor(batch["chosen_labels"]) + ) + + return batch + + @staticmethod + def concatenated_inputs( + batch: dict[str, list | torch.LongTensor], + is_encoder_decoder: bool = False, + label_pad_token_id: int = -100, + padding_value: int = 0, + device: torch.device | None = None, + ) -> dict[str, torch.LongTensor]: + """Concatenate the chosen and rejected inputs into a single tensor. + + Args: + batch: + A batch of data. Must contain the keys 'chosen_input_ids' and 'rejected_input_ids', which are tensors + of shape (batch_size, sequence_length). + is_encoder_decoder: + Whether the model is an encoder-decoder model. + label_pad_token_id: + The label pad token id. + padding_value: + The padding value to use for the concatenated inputs_ids. + device: + The device for the concatenated inputs. + + Returns: + A dictionary containing the concatenated inputs under the key 'concatenated_input_ids'. + """ + concatenated_batch = {} + + if is_encoder_decoder: + max_length = max(batch["chosen_labels"].shape[1], batch["rejected_labels"].shape[1]) + else: + max_length = max(batch["chosen_input_ids"].shape[1], batch["rejected_input_ids"].shape[1]) + + for k in batch: + if k.startswith("chosen") and isinstance(batch[k], torch.Tensor): + if "labels" in k or is_encoder_decoder: + pad_value = label_pad_token_id + elif k.endswith("_input_ids"): + pad_value = padding_value + elif k.endswith("_attention_mask"): + pad_value = 0 + concatenated_key = k.replace("chosen", "concatenated") + concatenated_batch[concatenated_key] = pad_to_length(batch[k], max_length, pad_value=pad_value) + for k in batch: + if k.startswith("rejected") and isinstance(batch[k], torch.Tensor): + if "labels" in k or is_encoder_decoder: + pad_value = label_pad_token_id + elif k.endswith("_input_ids"): + pad_value = padding_value + elif k.endswith("_attention_mask"): + pad_value = 0 + concatenated_key = k.replace("rejected", "concatenated") + concatenated_batch[concatenated_key] = torch.cat( + ( + concatenated_batch[concatenated_key], + pad_to_length(batch[k], max_length, pad_value=pad_value), + ), + dim=0, + ).to(device=device) + + if is_encoder_decoder: + concatenated_batch["concatenated_input_ids"] = batch["prompt_input_ids"].repeat(2, 1).to(device=device) + concatenated_batch["concatenated_attention_mask"] = ( + batch["prompt_attention_mask"].repeat(2, 1).to(device=device) + ) + + return concatenated_batch + + def cpo_loss( + self, + policy_chosen_logps: torch.FloatTensor, + policy_rejected_logps: torch.FloatTensor, + ) -> tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: + """Compute the CPO loss for a batch of policy and reference model log probabilities. + + Args: + policy_chosen_logps: + Log probabilities of the policy model for the chosen responses. Shape: (batch_size,) + policy_rejected_logps: + Log probabilities of the policy model for the rejected responses. Shape: (batch_size,) + + Returns: + A tuple of three tensors: (losses, chosen_rewards, rejected_rewards). The losses tensor contains the CPO + loss for each example in the batch. The chosen_rewards and rejected_rewards tensors contain the rewards for + the chosen and rejected responses, respectively. + """ + # Apply AlphaPO reward transformation if alpha != 0 + if self.alpha != 0.0: + # Compute probabilities + chosen_probs = torch.exp(policy_chosen_logps) + rejected_probs = torch.exp(policy_rejected_logps) + + # Apply AlphaPO transformation: r = (1 - p^(-alpha)) / alpha + policy_chosen_rewards = (1 - chosen_probs.pow(-self.alpha)) / self.alpha + policy_rejected_rewards = (1 - rejected_probs.pow(-self.alpha)) / self.alpha + + logits = (policy_chosen_rewards - policy_rejected_rewards).to(self.accelerator.device) + else: + # Standard log probability rewards when alpha = 0 + logits = (policy_chosen_logps - policy_rejected_logps).to(self.accelerator.device) + + # The beta is a temperature parameter for the CPO loss, typically something in the range of 0.1 to 0.5. + # We ignore the reference model as beta -> 0. The label_smoothing parameter encodes our uncertainty about the labels and + # calculates a conservative CPO loss. + + if self.loss_type == "simpo": + gamma_logratios = self.simpo_gamma / self.beta + logits = logits - gamma_logratios + # This reduces to Equation 3 from the CPO paper when label_smoothing -> 0. + losses = ( + -F.logsigmoid(self.beta * logits) * (1 - self.label_smoothing) + - F.logsigmoid(-self.beta * logits) * self.label_smoothing + ) + elif self.loss_type == "sigmoid": + # This reduces to Equation 3 from the CPO paper when label_smoothing -> 0. + losses = ( + -F.logsigmoid(self.beta * logits) * (1 - self.label_smoothing) + - F.logsigmoid(-self.beta * logits) * self.label_smoothing + ) + elif self.loss_type == "hinge": + losses = torch.relu(1 - self.beta * logits) + elif self.loss_type == "ipo": + # eqn (17) of the paper where beta is the regularization parameter for the IPO loss, denoted by tau in the paper. + losses = (logits - 1 / (2 * self.beta)) ** 2 + else: + raise ValueError( + f"Unknown loss type: {self.loss_type}. Should be one of ['sigmoid', 'hinge', 'ipo', 'simpo']" + ) + + # Calculate rewards for logging + if self.alpha != 0.0: + # When using AlphaPO transformation, use the transformed rewards + chosen_rewards = self.beta * policy_chosen_rewards.to(self.accelerator.device).detach() + rejected_rewards = self.beta * policy_rejected_rewards.to(self.accelerator.device).detach() + else: + # Standard log probability rewards + chosen_rewards = self.beta * (policy_chosen_logps.to(self.accelerator.device)).detach() + rejected_rewards = self.beta * (policy_rejected_logps.to(self.accelerator.device)).detach() + + return losses, chosen_rewards, rejected_rewards + + @staticmethod + def get_batch_logps( + logits: torch.FloatTensor, + labels: torch.LongTensor, + average_log_prob: bool = False, + label_pad_token_id: int = -100, + is_encoder_decoder: bool = False, + ) -> torch.FloatTensor: + """Compute the log probabilities of the given labels under the given logits. + + Args: + logits: Logits of the model (unnormalized). Shape: (batch_size, sequence_length, vocab_size) + labels: + Labels for which to compute the log probabilities. Label tokens with a value of label_pad_token_id are + ignored. Shape: (batch_size, sequence_length) + average_log_prob: + If True, return the average log probability per (non-masked) token. Otherwise, return the sum of the + log probabilities of the (non-masked) tokens. + label_pad_token_id: The label pad token id. + is_encoder_decoder: Whether the model is an encoder-decoder model. + + Returns: + A tensor of shape (batch_size,) containing the average/sum log probabilities of the given labels under the + given logits. + """ + if logits.shape[:-1] != labels.shape: + raise ValueError("Logits (batch and sequence length dim) and labels must have the same shape.") + + if not is_encoder_decoder: + labels = labels[:, 1:].clone() + logits = logits[:, :-1, :] + loss_mask = labels != label_pad_token_id + + # dummy token; we'll ignore the losses on these tokens later + labels[labels == label_pad_token_id] = 0 + + per_token_logps = selective_log_softmax(logits, labels) + + if average_log_prob: + return (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1) + else: + return (per_token_logps * loss_mask).sum(-1) + + def concatenated_forward( + self, model: nn.Module, batch: dict[str, list | torch.LongTensor] + ) -> tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: + """Run the given model on the given batch of inputs, concatenating the chosen and rejected inputs together. + + We do this to avoid doing two forward passes, because it's faster for FSDP. + """ + concatenated_batch = self.concatenated_inputs( + batch, + is_encoder_decoder=self.is_encoder_decoder, + label_pad_token_id=self.label_pad_token_id, + padding_value=self.padding_value, + device=self.accelerator.device, + ) + len_chosen = batch["chosen_labels"].shape[0] + + model_kwargs = ( + { + "decoder_input_ids": self._shift_right(concatenated_batch["concatenated_labels"]), + } + if self.is_encoder_decoder + else {} + ) + + if self.aux_loss_enabled: + model_kwargs["output_router_logits"] = True + + outputs = model( + concatenated_batch["concatenated_input_ids"], + attention_mask=concatenated_batch["concatenated_attention_mask"], + use_cache=False, + **model_kwargs, + ) + all_logits = outputs.logits + + def cross_entropy_loss(logits, labels): + if not self.is_encoder_decoder: + # Shift so that tokens < n predict n + logits = logits[..., :-1, :].contiguous() + labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = nn.CrossEntropyLoss() + logits = logits.view(-1, logits.shape[-1]) + labels = labels.view(-1) + # Enable model parallelism + labels = labels.to(logits.device) + loss = loss_fct(logits, labels) + return loss + + labels = concatenated_batch["concatenated_labels"].clone() + + if self.cpo_alpha == 0: + nll_loss = torch.tensor(0.0).to(self.accelerator.device) + else: + nll_loss = cross_entropy_loss(all_logits[:len_chosen], labels[:len_chosen]) + + all_logps = self.get_batch_logps( + all_logits, + concatenated_batch["concatenated_labels"], + average_log_prob=self.loss_type in ["ipo", "simpo"], + is_encoder_decoder=self.is_encoder_decoder, + label_pad_token_id=self.label_pad_token_id, + ) + + chosen_logps = all_logps[:len_chosen] + rejected_logps = all_logps[len_chosen:] + + chosen_logits = all_logits[:len_chosen] + rejected_logits = all_logits[len_chosen:] + + if self.aux_loss_enabled: + return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss, outputs.aux_loss) + + return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss) + + def get_batch_loss_metrics( + self, + model, + batch: dict[str, list | torch.LongTensor], + train_eval: Literal["train", "eval"] = "train", + ): + """Compute the CPO loss and other metrics for the given batch of inputs for train or test.""" + metrics = {} + + forward_output = self.concatenated_forward(model, batch) + ( + policy_chosen_logps, + policy_rejected_logps, + policy_chosen_logits, + policy_rejected_logits, + policy_nll_loss, + ) = forward_output[:5] + if self.aux_loss_enabled: + aux_loss = forward_output[5] + + losses, chosen_rewards, rejected_rewards = self.cpo_loss( + policy_chosen_logps, + policy_rejected_logps, + ) + + loss = losses.mean() + self.cpo_alpha * policy_nll_loss + reward_accuracies = (chosen_rewards > rejected_rewards).float() + + prefix = "eval_" if train_eval == "eval" else "" + metrics[f"{prefix}rewards/chosen"] = self.accelerator.gather_for_metrics(chosen_rewards).mean().item() + metrics[f"{prefix}rewards/rejected"] = self.accelerator.gather_for_metrics(rejected_rewards).mean().item() + metrics[f"{prefix}rewards/accuracies"] = self.accelerator.gather_for_metrics(reward_accuracies).mean().item() + metrics[f"{prefix}rewards/margins"] = ( + self.accelerator.gather_for_metrics(chosen_rewards - rejected_rewards).mean().item() + ) + metrics[f"{prefix}logps/rejected"] = ( + self.accelerator.gather_for_metrics(policy_rejected_logps).detach().mean().item() + ) + metrics[f"{prefix}logps/chosen"] = ( + self.accelerator.gather_for_metrics(policy_chosen_logps).detach().mean().item() + ) + metrics[f"{prefix}logits/rejected"] = ( + self.accelerator.gather_for_metrics(policy_rejected_logits.detach().mean()).mean().item() + ) + metrics[f"{prefix}logits/chosen"] = ( + self.accelerator.gather_for_metrics(policy_chosen_logits.detach().mean()).mean().item() + ) + metrics[f"{prefix}nll_loss"] = self.accelerator.gather_for_metrics(policy_nll_loss).detach().mean().item() + + if self.aux_loss_enabled: + loss += self.aux_loss_coef * aux_loss + + return loss, metrics + + def compute_loss( + self, + model: PreTrainedModel | nn.Module, + inputs: dict[str, torch.Tensor | Any], + return_outputs=False, + num_items_in_batch=None, + ) -> torch.Tensor | tuple[torch.Tensor, dict[str, torch.Tensor]]: + compute_loss_context_manager = ( + autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() + ) + + with compute_loss_context_manager: + loss, metrics = self.get_batch_loss_metrics(model, inputs, train_eval="train") + + # force log the metrics + self.store_metrics(metrics, train_eval="train") + + if return_outputs: + return (loss, metrics) + return loss + + def generate_from_model(self, model, batch: dict[str, torch.LongTensor]) -> str: + """Generate samples from the model and reference model for the given batch of inputs.""" + + # If one uses `generate_during_eval` with peft + bf16, we need to explicitly call generate with + # the torch amp context manager as some hidden states are silently casted to full precision. + generate_context_manager = ( + autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() + ) + + with generate_context_manager: + policy_output = model.generate( + input_ids=batch["prompt_input_ids"], + attention_mask=batch["prompt_attention_mask"], + max_length=self.max_length, + do_sample=True, + pad_token_id=self.processing_class.pad_token_id, + ) + + policy_output = pad_to_length(policy_output, self.max_length, self.processing_class.pad_token_id) + policy_output_decoded = self.processing_class.batch_decode(policy_output, skip_special_tokens=True) + + return policy_output_decoded + + def prediction_step( + self, + model: PreTrainedModel | nn.Module, + inputs: dict[str, torch.Tensor | Any], + prediction_loss_only: bool, + ignore_keys: list[str] | None = None, + ): + if ignore_keys is None: + if hasattr(model, "config"): + ignore_keys = getattr(model.config, "keys_to_ignore_at_inference", []) + else: + ignore_keys = [] + + prediction_context_manager = ( + autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() + ) + + with torch.no_grad(), prediction_context_manager: + loss, metrics = self.get_batch_loss_metrics(model, inputs, train_eval="eval") + + # force log the metrics + self.store_metrics(metrics, train_eval="eval") + + if prediction_loss_only: + return (loss.detach(), None, None) + + # logits for the chosen and rejected samples from model + logits_dict = { + "eval_logits/chosen": metrics["eval_logits/chosen"], + "eval_logits/rejected": metrics["eval_logits/rejected"], + } + logits = [v for k, v in logits_dict.items() if k not in ignore_keys] + logits = torch.tensor(logits, device=self.accelerator.device) + labels = torch.zeros(logits.shape[0], device=self.accelerator.device) + + return (loss.detach(), logits, labels) + + def store_metrics(self, metrics: dict[str, float], train_eval: Literal["train", "eval"] = "train") -> None: + for key, value in metrics.items(): + self._stored_metrics[train_eval][key].append(value) + + def evaluation_loop( + self, + dataloader: DataLoader, + description: str, + prediction_loss_only: bool | None = None, + ignore_keys: list[str] | None = None, + metric_key_prefix: str = "eval", + ) -> EvalLoopOutput: + """ + Overriding built-in evaluation loop to store metrics for each batch. Prediction/evaluation loop, shared by + `Trainer.evaluate()` and `Trainer.predict()`. + + Works both with or without labels. + """ + + # Sample and save to game log if requested (for one batch to save time) + if self.generate_during_eval: + # Generate random indices within the range of the total number of samples + num_samples = len(dataloader.dataset) + random_indices = random.sample(range(num_samples), k=self.args.eval_batch_size) + + # Use dataloader.dataset.select to get the random batch without iterating over the DataLoader + random_batch_dataset = dataloader.dataset.select(random_indices) + random_batch = self.data_collator(random_batch_dataset) + random_batch = self._prepare_inputs(random_batch) + + policy_output_decoded = self.generate_from_model(self.model, random_batch) + + table = pd.DataFrame( + columns=["Prompt", "Policy"], + data=[ + [prompt, pol[len(prompt) :]] + for prompt, pol in zip(random_batch["prompt"], policy_output_decoded, strict=True) + ], + ) + if "wandb" in self.args.report_to: + wandb.log({"game_log": wandb.Table(data=table)}) + + if "comet_ml" in self.args.report_to: + log_table_to_comet_experiment( + name="game_log.csv", + table=table, + ) + + # Base evaluation + initial_output = super().evaluation_loop( + dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix + ) + + return initial_output + + def log(self, logs: dict[str, float], start_time: float | None = None) -> None: + """ + Log `logs` on the various objects watching training, including stored metrics. + + Args: + logs (`dict[str, float]`): + The values to log. + start_time (`float`, *optional*): + Start time of the training. + """ + # logs either has 'loss' or 'eval_loss' + train_eval = "train" if "loss" in logs else "eval" + # Add averaged stored metrics to logs + for key, metrics in self._stored_metrics[train_eval].items(): + logs[key] = torch.tensor(metrics).mean().item() + del self._stored_metrics[train_eval] + return super().log(logs, start_time) + + def _shift_right(self, input_ids): + if self.decoder_start_token_id is None: + raise ValueError( + "model.config.decoder_start_token_id has to be defined. It is usually set to the pad_token_id." + ) + + # shift inputs to the right + if is_torch_fx_proxy(input_ids): + # Item assignment is not supported natively for proxies. + shifted_input_ids = torch.full(input_ids.shape[:-1] + (1,), self.decoder_start_token_id) + shifted_input_ids = torch.cat([shifted_input_ids, input_ids[..., :-1]], dim=-1) + else: + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids[..., 0] = self.decoder_start_token_id + + if self.pad_token_id is None: + raise ValueError("model.config.pad_token_id has to be defined.") + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, self.pad_token_id) + + return shifted_input_ids + + # Ensure the model card is saved along with the checkpoint + def _save_checkpoint(self, model, trial): + if self.args.hub_model_id is None: + model_name = Path(self.args.output_dir).name + else: + model_name = self.args.hub_model_id.split("/")[-1] + self.create_model_card(model_name=model_name) + super()._save_checkpoint(model, trial) diff --git a/trl/trainer/cpo_config.py b/trl/trainer/cpo_config.py index 426b00da153..268224e857a 100644 --- a/trl/trainer/cpo_config.py +++ b/trl/trainer/cpo_config.py @@ -12,207 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass, field -from typing import Any +import warnings +from dataclasses import dataclass -from transformers import TrainingArguments +from ..experimental.cpo import CPOConfig as _CPOConfig @dataclass -class CPOConfig(TrainingArguments): +class CPOConfig(_CPOConfig): r""" Configuration class for the [`CPOTrainer`]. - This class includes only the parameters that are specific to CPO training. For a full list of training arguments, - please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may - differ from those in [`~transformers.TrainingArguments`]. - - Using [`~transformers.HfArgumentParser`] we can turn this class into - [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the - command line. - - Parameters: - max_length (`int` or `None`, *optional*, defaults to `1024`): - Maximum length of the sequences (prompt + completion) in the batch. This argument is required if you want - to use the default data collator. - max_prompt_length (`int` or `None`, *optional*, defaults to `512`): - Maximum length of the prompt. This argument is required if you want to use the default data collator. - max_completion_length (`int`, *optional*): - Maximum length of the completion. This argument is required if you want to use the default data collator - and your model is an encoder-decoder. - beta (`float`, *optional*, defaults to `0.1`): - Parameter controlling the deviation from the reference model. Higher β means less deviation from the - reference model. For the IPO loss (`loss_type="ipo"`), β is the regularization parameter denoted by τ in - the [paper](https://huggingface.co/papers/2310.12036). - label_smoothing (`float`, *optional*, defaults to `0.0`): - Label smoothing factor. This argument is required if you want to use the default data collator. - loss_type (`str`, *optional*, defaults to `"sigmoid"`): - Type of loss to use. Possible values are: - - - `"sigmoid"`: sigmoid loss from the original [DPO](https://huggingface.co/papers/2305.18290) paper. - - `"hinge"`: hinge loss on the normalized likelihood from the - [SLiC](https://huggingface.co/papers/2305.10425) paper. - - `"ipo"`: IPO loss from the [IPO](https://huggingface.co/papers/2310.12036) paper. - - `"simpo"`: SimPO loss from the [SimPO](https://huggingface.co/papers/2405.14734) paper. - - `"alphapo"`: AlphaPO loss from the [AlphaPO](https://huggingface.co/papers/2501.03884) paper. This - automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`. - - disable_dropout (`bool`, *optional*, defaults to `True`): - Whether to disable dropout in the model. - cpo_alpha (`float`, *optional*, defaults to `1.0`): - Weight of the BC regularizer in CPO training. - simpo_gamma (`float`, *optional*, defaults to `0.5`): - Target reward margin for the SimPO loss, used only when the `loss_type="simpo"`. - alpha (`float`, *optional*, defaults to `0.0`): - Alpha parameter that controls reward function shape across all loss types. When alpha=0 (default), uses - standard log probability rewards. When `alpha != 0`, applies AlphaPO transformation: `r = (1 - p^(-alpha)) - / alpha` from the [AlphaPO paper](https://huggingface.co/papers/2501.03884). This parameter works with all - loss types. - label_pad_token_id (`int`, *optional*, defaults to `-100`): - Label pad token id. This argument is required if you want to use the default data collator. - padding_value (`int`, *optional*): - Padding value to use. If `None`, the padding value of the tokenizer is used. - truncation_mode (`str`,*optional*, defaults to `"keep_end"`): - Truncation mode to use when the prompt is too long. Possible values are `"keep_end"` or `"keep_start"`. - This argument is required if you want to use the default data collator. - generate_during_eval (`bool`, *optional*, defaults to `False`): - If `True`, generates and logs completions from the model to W&B or Comet during evaluation. - is_encoder_decoder (`bool`, *optional*): - When using the `model_init` argument (callable) to instantiate the model instead of the `model` argument, - you need to specify if the model returned by the callable is an encoder-decoder model. - model_init_kwargs (`dict[str, Any]`, *optional*): - Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the model from a - string. - dataset_num_proc (`int`, *optional*): - Number of processes to use for processing the dataset. + This class is now located in `trl.experimental.cpo`. Please update your imports to + `from trl.experimental.cpo import CPOConfig`. The current import path will be removed in TRL 0.29. + For more information, see https://github.com/huggingface/trl/issues/4223. """ - _VALID_DICT_FIELDS = TrainingArguments._VALID_DICT_FIELDS + ["model_init_kwargs"] - - # Parameters whose default values are overridden from TrainingArguments - learning_rate: float = field( - default=1e-6, - metadata={"help": "The initial learning rate for AdamW."}, - ) - logging_steps: float = field( - default=10, - metadata={ - "help": "Log every X updates steps. Should be an integer or a float in range `[0,1)`. If smaller than 1, " - "will be interpreted as ratio of total training steps." - }, - ) - gradient_checkpointing: bool = field( - default=True, - metadata={ - "help": "If True, use gradient checkpointing to save memory at the expense of slower backward pass." - }, - ) - bf16: bool | None = field( - default=None, - metadata={ - "help": "Whether to use bf16 (mixed) precision instead of 32-bit. Requires Ampere or higher NVIDIA " - "architecture or Intel XPU or using CPU (use_cpu) or Ascend NPU. If not set, it defaults to `True` if " - "`fp16` is not set." - }, - ) - - max_length: int | None = field( - default=1024, - metadata={"help": "Maximum length of the sequences (prompt + completion) in the batch."}, - ) - max_prompt_length: int | None = field( - default=512, - metadata={ - "help": "Maximum length of the prompt. This argument is required if you want to use the default data " - "collator and your model is an encoder-decoder." - }, - ) - max_completion_length: int | None = field( - default=None, - metadata={ - "help": "Maximum length of the completion. This argument is required if you want to use the default data " - "collator and your model is an encoder-decoder." - }, - ) - beta: float = field( - default=0.1, - metadata={ - "help": "Parameter controlling the deviation from the reference model. Higher β means less deviation from " - "the reference model." - }, - ) - label_smoothing: float = field( - default=0.0, - metadata={"help": "Label smoothing factor."}, - ) - loss_type: str = field( - default="sigmoid", - metadata={ - "help": "Type of loss to use.", - "choices": ["sigmoid", "hinge", "ipo", "simpo", "alphapo"], - }, - ) - disable_dropout: bool = field( - default=True, - metadata={"help": "Whether to disable dropout in the model."}, - ) - cpo_alpha: float = field( - default=1.0, - metadata={"help": "Weight of the BC regularizer in CPO training."}, - ) - simpo_gamma: float = field( - default=0.5, - metadata={"help": "Target reward margin for the SimPO loss, used only when the `loss_type='simpo'`."}, - ) - alpha: float = field( - default=0.0, - metadata={ - "help": "Alpha parameter that controls reward function shape across all loss types. When alpha=0 " - "(default), uses standard log probability rewards. When `alpha != 0`, applies AlphaPO transformation: " - "`r = (1 - p^(-alpha)) / alpha` from the AlphaPO paper. This parameter works with all loss types." - }, - ) - label_pad_token_id: int = field( - default=-100, - metadata={"help": "Label pad token id."}, - ) - padding_value: int | None = field( - default=None, - metadata={"help": "Padding value to use. If `None`, the padding value of the tokenizer is used."}, - ) - truncation_mode: str = field( - default="keep_end", - metadata={ - "help": "Truncation mode to use when the prompt is too long.", - "choices": ["keep_end", "keep_start"], - }, - ) - generate_during_eval: bool = field( - default=False, - metadata={"help": "If `True`, generates and logs completions from the model to W&B during evaluation."}, - ) - is_encoder_decoder: bool | None = field( - default=None, - metadata={"help": "Whether the model is an encoder-decoder model."}, - ) - model_init_kwargs: dict[str, Any] | None = field( - default=None, - metadata={ - "help": "Keyword arguments to pass to `AutoModelForCausalLM.from_pretrained` when instantiating the model " - "from a string." - }, - ) - dataset_num_proc: int | None = field( - default=None, - metadata={"help": "Number of processes to use for processing the dataset."}, - ) - def __post_init__(self): - self.bf16 = not (self.fp16) if self.bf16 is None else self.bf16 - - # Syntactic sugar for AlphaPO: set loss_type to "simpo" and cpo_alpha to 0.0 - if self.loss_type == "alphapo": - self.loss_type = "simpo" - self.cpo_alpha = 0.0 - + warnings.warn( + "The `CPOConfig` is now located in `trl.experimental`. Please update your imports to " + "`from trl.experimental.cpo import CPOConfig`. The current import path will be removed and no longer " + "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223.", + FutureWarning, + stacklevel=2, + ) super().__post_init__() diff --git a/trl/trainer/cpo_trainer.py b/trl/trainer/cpo_trainer.py index 565983a6a10..3ce934d686b 100644 --- a/trl/trainer/cpo_trainer.py +++ b/trl/trainer/cpo_trainer.py @@ -12,71 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect -import os -import random -import textwrap import warnings -from collections import defaultdict from collections.abc import Callable -from contextlib import nullcontext -from pathlib import Path -from typing import Any, Literal -import numpy as np -import pandas as pd -import torch import torch.nn as nn -import torch.nn.functional as F -from accelerate import PartialState, logging from datasets import Dataset -from torch import autocast -from torch.utils.data import DataLoader from transformers import ( - AutoModelForCausalLM, BaseImageProcessor, DataCollator, FeatureExtractionMixin, PreTrainedModel, PreTrainedTokenizerBase, ProcessorMixin, - is_comet_available, - is_wandb_available, ) from transformers.trainer_callback import TrainerCallback from transformers.trainer_utils import EvalLoopOutput -from transformers.utils import is_peft_available, is_torch_fx_proxy - -from ..data_utils import maybe_apply_chat_template, maybe_extract_prompt -from .base_trainer import BaseTrainer -from .cpo_config import CPOConfig -from .utils import ( - DPODataCollatorWithPadding, - add_bos_token_if_needed, - add_eos_token_if_needed, - disable_dropout_in_model, - log_table_to_comet_experiment, - pad_to_length, - peft_module_casting_to_bf16, - selective_log_softmax, -) - - -if is_peft_available(): - from peft import PeftModel, get_peft_model, prepare_model_for_kbit_training - - -if is_wandb_available(): - import wandb +from ..experimental.cpo import CPOConfig, CPOTrainer as _CPOTrainer -logger = logging.get_logger(__name__) - -class CPOTrainer(BaseTrainer): +class CPOTrainer(_CPOTrainer): r""" Initialize CPOTrainer. + This class is now located in `trl.experimental.cpo`. Please update your imports to + `from trl.experimental.cpo import CPOTrainer`. The current import path will be removed in TRL 0.29. + For more information, see https://github.com/huggingface/trl/issues/4223. + Args: model ([`~transformers.PreTrainedModel`]): The model to train, preferably an [`~transformers.AutoModelForSequenceClassification`]. @@ -111,23 +73,6 @@ class CPOTrainer(BaseTrainer): metric values. """ - _tag_names = ["trl", "cpo"] - _name = "CPO" - _paper = { - "title": "Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation", - "id": "2401.08417", - # docstyle-ignore - "citation": textwrap.dedent("""\ - @inproceedings{xu2024contrastive, - title = {{Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation}}, - author = {Haoran Xu and Amr Sharaf and Yunmo Chen and Weiting Tan and Lingfeng Shen and Benjamin Van Durme and Kenton Murray and Young Jin Kim}, - year = 2024, - booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}, - publisher = {OpenReview.net}, - url = {https://openreview.net/forum?id=51iwkioZpn} - }"""), - } - def __init__( self, model: PreTrainedModel | nn.Module | str | None = None, @@ -147,233 +92,13 @@ def __init__( peft_config: dict | None = None, compute_metrics: Callable[[EvalLoopOutput], dict] | None = None, ): - if not os.environ.get("TRL_EXPERIMENTAL_SILENCE"): - warnings.warn( - "This trainer will soon be moved to trl.experimental and is a candidate for removal. If you rely on " - "it and want it to remain, please share your comments here: " - "https://github.com/huggingface/trl/issues/4223. Silence this warning by setting environment variable " - "TRL_EXPERIMENTAL_SILENCE=1." - ) - if args.model_init_kwargs is None: - model_init_kwargs = {} - elif not isinstance(model, str): - raise ValueError("You passed model_kwargs to the CPOTrainer. But your model is already instantiated.") - else: - model_init_kwargs = args.model_init_kwargs - dtype = model_init_kwargs.get("dtype") - if dtype is not None: - # Convert to `torch.dtype` if an str is passed - if isinstance(dtype, str) and dtype != "auto": - dtype = getattr(torch, dtype) - if dtype != "auto" and not isinstance(dtype, torch.dtype): - raise ValueError( - f"Invalid `dtype` passed to the CPOConfig. Expected a string with either `torch.dtype` or 'auto', but got {dtype}." - ) - model_init_kwargs["dtype"] = dtype - - if isinstance(model, str): - model = AutoModelForCausalLM.from_pretrained(model, **model_init_kwargs) - - # Initialize this variable to False. This helps tracking the case when `peft_module_casting_to_bf16` - # has been called in order to properly call autocast if needed. - self._peft_has_been_casted_to_bf16 = False - - if not is_peft_available() and peft_config is not None: - raise ValueError( - "PEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT models" - ) - elif is_peft_available() and peft_config is not None: - # if model is a peft model and we have a peft_config, we merge and unload it first - if isinstance(model, PeftModel): - model = model.merge_and_unload() - - if getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_loaded_in_4bit", False): - _support_gc_kwargs = hasattr( - args, "gradient_checkpointing_kwargs" - ) and "gradient_checkpointing_kwargs" in list( - inspect.signature(prepare_model_for_kbit_training).parameters - ) - - prepare_model_kwargs = {"use_gradient_checkpointing": args.gradient_checkpointing} - - if _support_gc_kwargs: - prepare_model_kwargs["gradient_checkpointing_kwargs"] = args.gradient_checkpointing_kwargs - - model = prepare_model_for_kbit_training(model, **prepare_model_kwargs) - elif args.gradient_checkpointing: - # For backward compatibility with older versions of transformers - if hasattr(model, "enable_input_require_grads"): - model.enable_input_require_grads() - else: - - def make_inputs_require_grad(module, input, output): - output.requires_grad_(True) - - model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) - - # get peft model with the given config - model = get_peft_model(model, peft_config) - if args.bf16 and getattr(model, "is_loaded_in_4bit", False): - peft_module_casting_to_bf16(model) - # If args.bf16 we need to explicitly call `generate` with torch amp autocast context manager - self._peft_has_been_casted_to_bf16 = True - - # For models that use gradient_checkpointing, we need to attach a hook that enables input - # to explicitly have `requires_grad=True`, otherwise training will either silently - # fail or completely fail. - elif args.gradient_checkpointing: - # For backward compatibility with older versions of transformers - if hasattr(model, "enable_input_require_grads"): - model.enable_input_require_grads() - else: - - def make_inputs_require_grad(module, input, output): - output.requires_grad_(True) - - model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) - - if args.generate_during_eval and not (is_wandb_available() or is_comet_available()): - raise ValueError( - "`generate_during_eval=True` requires Weights and Biases or Comet to be installed." - " Please install `wandb` or `comet-ml` to resolve." - ) - - if model is not None: - self.is_encoder_decoder = model.config.is_encoder_decoder - elif args.is_encoder_decoder is None: - raise ValueError("When no model is provided, you need to pass the parameter is_encoder_decoder.") - else: - self.is_encoder_decoder = args.is_encoder_decoder - - if self.is_encoder_decoder: - self.decoder_start_token_id = model.config.decoder_start_token_id - self.pad_token_id = model.config.pad_token_id - - if processing_class is None: - raise ValueError("processing_class must be specified to tokenize a CPO dataset.") - if args.max_length is None: - logger.warning( - "`max_length` is not set in the CPOConfig's init" - " it will default to `512` by default, but you should do it yourself in the future.", - ) - max_length = 512 - else: - max_length = args.max_length - if args.max_prompt_length is None: - logger.warning( - "`max_prompt_length` is not set in the CPOConfig's init" - " it will default to `128` by default, but you should do it yourself in the future.", - ) - max_prompt_length = 128 - else: - max_prompt_length = args.max_prompt_length - - if not max_prompt_length < max_length: - raise ValueError( - f"max_prompt_length ({max_prompt_length}) should be strictly less than max_length ({max_length})." - ) - - if args.max_completion_length is None and self.is_encoder_decoder: - logger.warning( - "When using an encoder decoder architecture, you should set `max_completion_length` in the CPOConfig's init" - " it will default to `128` by default, but you should do it yourself in the future.", - ) - max_completion_length = 128 - else: - max_completion_length = args.max_completion_length - - if data_collator is None: - data_collator = DPODataCollatorWithPadding( - pad_token_id=processing_class.pad_token_id, - label_pad_token_id=args.label_pad_token_id, - is_encoder_decoder=self.is_encoder_decoder, - ) - - if args.remove_unused_columns: - args.remove_unused_columns = False - # warn users - logger.warning( - "When using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments" - " we have set it for you, but you should do it yourself in the future.", - ) - - self.use_dpo_data_collator = True - else: - self.use_dpo_data_collator = False - - # Disable dropout in the model - if args.disable_dropout: - disable_dropout_in_model(model) - - self.max_length = max_length - self.generate_during_eval = args.generate_during_eval - self.label_pad_token_id = args.label_pad_token_id - self.padding_value = args.padding_value if args.padding_value is not None else processing_class.pad_token_id - self.max_prompt_length = max_prompt_length - self.truncation_mode = args.truncation_mode - self.max_completion_length = max_completion_length - self.processing_class = processing_class - - if args.loss_type in ["hinge", "ipo"] and args.label_smoothing > 0: - logger.warning( - f"You are using the {args.loss_type} loss type that does not support label smoothing. The " - "`label_smoothing` parameter will be ignored. Set `label_smoothing` to `0.0` to remove this warning.", - ) - if args.loss_type == "kto_pair": - raise ValueError("Support for kto_pair has been removed in CPOTrainer. Please use KTOTrainer.") - - self.beta = args.beta - self.label_smoothing = args.label_smoothing - self.loss_type = args.loss_type - self.cpo_alpha = args.cpo_alpha - self.aux_loss_enabled = getattr(model.config, "output_router_logits", False) - self.aux_loss_coef = getattr(model.config, "router_aux_loss_coef", 0.0) - if self.aux_loss_enabled and self.aux_loss_coef == 0.0: - logger.warning( - "You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to " - "`0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value " - "greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary " - "loss.", - ) - - if args.loss_type == "simpo": - self.simpo_gamma = args.simpo_gamma - - # AlphaPO parameter for reward shaping - self.alpha = args.alpha - - self._stored_metrics = defaultdict(lambda: defaultdict(list)) - - # The trainer estimates the number of FLOPs (floating-point operations) using the number of elements in the - # input tensor associated with the key "input_ids". However, in CPO, the sampled data does not include the - # "input_ids" key. Instead, the available keys are "prompt_input_ids", "chosen_input_ids", and - # "rejected_input_ids". As a result, the trainer issues the warning: "Could not estimate the number of tokens - # of the input, floating-point operations will not be computed." To suppress this warning, we set the - # "estimate_tokens" key in the model's "warnings_issued" dictionary to True. This acts as a flag to indicate - # that the warning has already been issued. - model.warnings_issued["estimate_tokens"] = True - - # Compute that only on the main process for faster data processing. - # see: https://github.com/huggingface/trl/pull/1255 - with PartialState().main_process_first(): - # Extract the prompt if needed, and apply the chat template if needed - train_dataset = train_dataset.map(maybe_extract_prompt, num_proc=args.dataset_num_proc) - train_dataset = train_dataset.map( - maybe_apply_chat_template, fn_kwargs={"tokenizer": processing_class}, num_proc=args.dataset_num_proc - ) - if eval_dataset is not None: - eval_dataset = eval_dataset.map(maybe_extract_prompt, num_proc=args.dataset_num_proc) - eval_dataset = eval_dataset.map( - maybe_apply_chat_template, - fn_kwargs={"tokenizer": processing_class}, - num_proc=args.dataset_num_proc, - ) - - # tokenize the dataset - train_dataset = train_dataset.map(self.tokenize_row, num_proc=args.dataset_num_proc) - if eval_dataset is not None: - eval_dataset = eval_dataset.map(self.tokenize_row, num_proc=args.dataset_num_proc) - + warnings.warn( + "The `CPOTrainer` is now located in `trl.experimental`. Please update your imports to " + "`from trl.experimental.cpo import CPOTrainer`. The current import path will be removed and no longer " + "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223.", + FutureWarning, + stacklevel=2, + ) super().__init__( model=model, args=args, @@ -382,716 +107,9 @@ def make_inputs_require_grad(module, input, output): eval_dataset=eval_dataset, processing_class=processing_class, model_init=model_init, - compute_metrics=compute_metrics, callbacks=callbacks, optimizers=optimizers, preprocess_logits_for_metrics=preprocess_logits_for_metrics, + peft_config=peft_config, + compute_metrics=compute_metrics, ) - - # Gradient accumulation requires scaled loss. Normally, loss scaling in the parent class depends on whether the - # model accepts loss-related kwargs. Since we compute our own loss, this check is irrelevant. We set - # self.model_accepts_loss_kwargs to False to enable scaling. - self.model_accepts_loss_kwargs = False - - # Add tags for models that have been loaded with the correct transformers version - if hasattr(self.model, "add_model_tags"): - self.model.add_model_tags(self._tag_names) - - if not hasattr(self, "accelerator"): - raise AttributeError( - "Your `Trainer` does not have an `accelerator` object. Consider upgrading `transformers`." - ) - - def build_tokenized_answer(self, prompt, answer): - """ - Llama tokenizer does satisfy `enc(a + b) = enc(a) + enc(b)`. It does ensure `enc(a + b) = enc(a) + enc(a + - b)[len(enc(a)):]`. Reference: - https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 - """ - - full_tokenized = self.processing_class(prompt + answer, add_special_tokens=False) - prompt_input_ids = self.processing_class(prompt, add_special_tokens=False)["input_ids"] - - answer_input_ids = full_tokenized["input_ids"][len(prompt_input_ids) :] - answer_attention_mask = full_tokenized["attention_mask"][len(prompt_input_ids) :] - - # Concat tokens to form `enc(a) + enc(a + b)[len(enc(a)):]` - full_concat_input_ids = np.concatenate([prompt_input_ids, answer_input_ids]) - - # Prepare input tokens for token by token comparison - full_input_ids = np.array(full_tokenized["input_ids"]) - - if len(full_input_ids) != len(full_concat_input_ids): - raise ValueError("Prompt input ids and answer input ids should have the same length.") - - # On some tokenizers, like Llama-2 tokenizer, there are occasions where tokens - # can be merged together when tokenizing prompt+answer. This could result - # on the last token from the prompt being different when tokenized on its own - # vs when done as prompt+answer. - response_token_ids_start_idx = len(prompt_input_ids) - - # If tokenized prompt is different than both prompt+answer, then it means the - # last token has changed due to merging. - if prompt_input_ids != full_tokenized["input_ids"][:response_token_ids_start_idx]: - response_token_ids_start_idx -= 1 - - prompt_input_ids = full_tokenized["input_ids"][:response_token_ids_start_idx] - prompt_attention_mask = full_tokenized["attention_mask"][:response_token_ids_start_idx] - - if len(prompt_input_ids) != len(prompt_attention_mask): - raise ValueError("Prompt input ids and attention mask should have the same length.") - - answer_input_ids = full_tokenized["input_ids"][response_token_ids_start_idx:] - answer_attention_mask = full_tokenized["attention_mask"][response_token_ids_start_idx:] - - return dict( - prompt_input_ids=prompt_input_ids, - prompt_attention_mask=prompt_attention_mask, - input_ids=answer_input_ids, - attention_mask=answer_attention_mask, - ) - - def tokenize_row(self, feature, model: PreTrainedModel | nn.Module | None = None) -> dict: - """Tokenize a single row from a CPO specific dataset. - - At this stage, we don't convert to PyTorch tensors yet; we just handle the truncation in case the prompt + - chosen or prompt + rejected responses is/are too long. First we truncate the prompt; if we're still too long, - we truncate the chosen/rejected. - - We also create the labels for the chosen/rejected responses, which are of length equal to the sum of the length - of the prompt and the chosen/rejected response, with label_pad_token_id for the prompt tokens. - """ - batch = {} - prompt = feature["prompt"] - chosen = feature["chosen"] - rejected = feature["rejected"] - - if not self.is_encoder_decoder: - # Check issues below for more details - # 1. https://github.com/huggingface/trl/issues/907 - # 2. https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 - # 3. https://github.com/LianjiaTech/BELLE/issues/337 - - if not isinstance(prompt, str): - raise ValueError(f"prompt should be an str but got {type(prompt)}") - prompt_tokens = self.processing_class(prompt, add_special_tokens=False) - prompt_tokens = {f"prompt_{k}": v for k, v in prompt_tokens.items()} - - if not isinstance(chosen, str): - raise ValueError(f"chosen should be an str but got {type(chosen)}") - chosen_tokens = self.build_tokenized_answer(prompt, chosen) - - if not isinstance(rejected, str): - raise ValueError(f"rejected should be an str but got {type(rejected)}") - rejected_tokens = self.build_tokenized_answer(prompt, rejected) - - # Last prompt token might get merged by tokenizer and - # it should not be included for generation if that happens - prompt_len_input_ids = len(prompt_tokens["prompt_input_ids"]) - - chosen_prompt_len_input_ids = len(chosen_tokens["prompt_input_ids"]) - rejected_prompt_len_input_ids = len(rejected_tokens["prompt_input_ids"]) - prompt_len_input_ids = min(chosen_prompt_len_input_ids, rejected_prompt_len_input_ids) - - for k, v in prompt_tokens.items(): - prompt_tokens[k] = v[:prompt_len_input_ids] - - # Make sure prompts only have one different token at most an - # and length only differs by 1 at most - num_diff_tokens = sum( - a != b - for a, b in zip(chosen_tokens["prompt_input_ids"], rejected_tokens["prompt_input_ids"], strict=True) - ) - num_diff_len = abs(chosen_prompt_len_input_ids - rejected_prompt_len_input_ids) - if num_diff_tokens > 1 or num_diff_len > 1: - raise ValueError( - "Chosen and rejected prompt_input_ids might only differ on the " - "last token due to tokenizer merge ops." - ) - - # add BOS token to head of prompt. Avoid adding if it's already there - prompt_tokens, chosen_tokens, rejected_tokens = add_bos_token_if_needed( - self.processing_class.bos_token_id, - prompt_len_input_ids, - prompt_tokens, - chosen_prompt_len_input_ids, - chosen_tokens, - rejected_prompt_len_input_ids, - rejected_tokens, - ) - - # add EOS token to end of answer. Avoid adding if it's already there - chosen_tokens, rejected_tokens = add_eos_token_if_needed( - self.processing_class.eos_token_id, chosen_tokens, rejected_tokens - ) - - longer_response_length = max(len(chosen_tokens["input_ids"]), len(rejected_tokens["input_ids"])) - - # if combined sequence is too long, truncate the prompt - for answer_tokens in [chosen_tokens, rejected_tokens, prompt_tokens]: - if len(answer_tokens["prompt_input_ids"]) + longer_response_length > self.max_length: - if self.truncation_mode == "keep_start": - for k in ["prompt_input_ids", "prompt_attention_mask"]: - answer_tokens[k] = answer_tokens[k][: self.max_prompt_length] - elif self.truncation_mode == "keep_end": - for k in ["prompt_input_ids", "prompt_attention_mask"]: - answer_tokens[k] = answer_tokens[k][-self.max_prompt_length :] - else: - raise ValueError(f"Unknown truncation mode: {self.truncation_mode}") - - # if that's still too long, truncate the response - for answer_tokens in [chosen_tokens, rejected_tokens]: - if len(answer_tokens["prompt_input_ids"]) + longer_response_length > self.max_length: - for k in ["input_ids", "attention_mask"]: - answer_tokens[k] = answer_tokens[k][: self.max_length - self.max_prompt_length] - - # Create labels - chosen_sequence_tokens = { - k: chosen_tokens[f"prompt_{k}"] + chosen_tokens[k] for k in ["input_ids", "attention_mask"] - } - rejected_sequence_tokens = { - k: rejected_tokens[f"prompt_{k}"] + rejected_tokens[k] for k in ["input_ids", "attention_mask"] - } - chosen_sequence_tokens["labels"] = chosen_sequence_tokens["input_ids"][:] - chosen_sequence_tokens["labels"][: len(chosen_tokens["prompt_input_ids"])] = [ - self.label_pad_token_id - ] * len(chosen_tokens["prompt_input_ids"]) - rejected_sequence_tokens["labels"] = rejected_sequence_tokens["input_ids"][:] - rejected_sequence_tokens["labels"][: len(rejected_tokens["prompt_input_ids"])] = [ - self.label_pad_token_id - ] * len(rejected_tokens["prompt_input_ids"]) - - for k, toks in { - "chosen_": chosen_sequence_tokens, - "rejected_": rejected_sequence_tokens, - "": prompt_tokens, - }.items(): - for type_key, tokens in toks.items(): - if type_key == "token_type_ids": - continue - batch[f"{k}{type_key}"] = tokens - - else: - chosen_tokens = self.processing_class( - chosen, truncation=True, max_length=self.max_completion_length, add_special_tokens=True - ) - rejected_tokens = self.processing_class( - rejected, truncation=True, max_length=self.max_completion_length, add_special_tokens=True - ) - prompt_tokens = self.processing_class( - prompt, truncation=True, max_length=self.max_prompt_length, add_special_tokens=True - ) - - batch["chosen_labels"] = chosen_tokens["input_ids"] - batch["rejected_labels"] = rejected_tokens["input_ids"] - batch["prompt_input_ids"] = prompt_tokens["input_ids"] - batch["prompt_attention_mask"] = prompt_tokens["attention_mask"] - - if model is not None and hasattr(model, "prepare_decoder_input_ids_from_labels"): - batch["rejected_decoder_input_ids"] = model.prepare_decoder_input_ids_from_labels( - labels=torch.tensor(batch["rejected_labels"]) - ) - batch["chosen_decoder_input_ids"] = model.prepare_decoder_input_ids_from_labels( - labels=torch.tensor(batch["chosen_labels"]) - ) - - return batch - - @staticmethod - def concatenated_inputs( - batch: dict[str, list | torch.LongTensor], - is_encoder_decoder: bool = False, - label_pad_token_id: int = -100, - padding_value: int = 0, - device: torch.device | None = None, - ) -> dict[str, torch.LongTensor]: - """Concatenate the chosen and rejected inputs into a single tensor. - - Args: - batch: - A batch of data. Must contain the keys 'chosen_input_ids' and 'rejected_input_ids', which are tensors - of shape (batch_size, sequence_length). - is_encoder_decoder: - Whether the model is an encoder-decoder model. - label_pad_token_id: - The label pad token id. - padding_value: - The padding value to use for the concatenated inputs_ids. - device: - The device for the concatenated inputs. - - Returns: - A dictionary containing the concatenated inputs under the key 'concatenated_input_ids'. - """ - concatenated_batch = {} - - if is_encoder_decoder: - max_length = max(batch["chosen_labels"].shape[1], batch["rejected_labels"].shape[1]) - else: - max_length = max(batch["chosen_input_ids"].shape[1], batch["rejected_input_ids"].shape[1]) - - for k in batch: - if k.startswith("chosen") and isinstance(batch[k], torch.Tensor): - if "labels" in k or is_encoder_decoder: - pad_value = label_pad_token_id - elif k.endswith("_input_ids"): - pad_value = padding_value - elif k.endswith("_attention_mask"): - pad_value = 0 - concatenated_key = k.replace("chosen", "concatenated") - concatenated_batch[concatenated_key] = pad_to_length(batch[k], max_length, pad_value=pad_value) - for k in batch: - if k.startswith("rejected") and isinstance(batch[k], torch.Tensor): - if "labels" in k or is_encoder_decoder: - pad_value = label_pad_token_id - elif k.endswith("_input_ids"): - pad_value = padding_value - elif k.endswith("_attention_mask"): - pad_value = 0 - concatenated_key = k.replace("rejected", "concatenated") - concatenated_batch[concatenated_key] = torch.cat( - ( - concatenated_batch[concatenated_key], - pad_to_length(batch[k], max_length, pad_value=pad_value), - ), - dim=0, - ).to(device=device) - - if is_encoder_decoder: - concatenated_batch["concatenated_input_ids"] = batch["prompt_input_ids"].repeat(2, 1).to(device=device) - concatenated_batch["concatenated_attention_mask"] = ( - batch["prompt_attention_mask"].repeat(2, 1).to(device=device) - ) - - return concatenated_batch - - def cpo_loss( - self, - policy_chosen_logps: torch.FloatTensor, - policy_rejected_logps: torch.FloatTensor, - ) -> tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: - """Compute the CPO loss for a batch of policy and reference model log probabilities. - - Args: - policy_chosen_logps: - Log probabilities of the policy model for the chosen responses. Shape: (batch_size,) - policy_rejected_logps: - Log probabilities of the policy model for the rejected responses. Shape: (batch_size,) - - Returns: - A tuple of three tensors: (losses, chosen_rewards, rejected_rewards). The losses tensor contains the CPO - loss for each example in the batch. The chosen_rewards and rejected_rewards tensors contain the rewards for - the chosen and rejected responses, respectively. - """ - # Apply AlphaPO reward transformation if alpha != 0 - if self.alpha != 0.0: - # Compute probabilities - chosen_probs = torch.exp(policy_chosen_logps) - rejected_probs = torch.exp(policy_rejected_logps) - - # Apply AlphaPO transformation: r = (1 - p^(-alpha)) / alpha - policy_chosen_rewards = (1 - chosen_probs.pow(-self.alpha)) / self.alpha - policy_rejected_rewards = (1 - rejected_probs.pow(-self.alpha)) / self.alpha - - logits = (policy_chosen_rewards - policy_rejected_rewards).to(self.accelerator.device) - else: - # Standard log probability rewards when alpha = 0 - logits = (policy_chosen_logps - policy_rejected_logps).to(self.accelerator.device) - - # The beta is a temperature parameter for the CPO loss, typically something in the range of 0.1 to 0.5. - # We ignore the reference model as beta -> 0. The label_smoothing parameter encodes our uncertainty about the labels and - # calculates a conservative CPO loss. - - if self.loss_type == "simpo": - gamma_logratios = self.simpo_gamma / self.beta - logits = logits - gamma_logratios - # This reduces to Equation 3 from the CPO paper when label_smoothing -> 0. - losses = ( - -F.logsigmoid(self.beta * logits) * (1 - self.label_smoothing) - - F.logsigmoid(-self.beta * logits) * self.label_smoothing - ) - elif self.loss_type == "sigmoid": - # This reduces to Equation 3 from the CPO paper when label_smoothing -> 0. - losses = ( - -F.logsigmoid(self.beta * logits) * (1 - self.label_smoothing) - - F.logsigmoid(-self.beta * logits) * self.label_smoothing - ) - elif self.loss_type == "hinge": - losses = torch.relu(1 - self.beta * logits) - elif self.loss_type == "ipo": - # eqn (17) of the paper where beta is the regularization parameter for the IPO loss, denoted by tau in the paper. - losses = (logits - 1 / (2 * self.beta)) ** 2 - else: - raise ValueError( - f"Unknown loss type: {self.loss_type}. Should be one of ['sigmoid', 'hinge', 'ipo', 'simpo']" - ) - - # Calculate rewards for logging - if self.alpha != 0.0: - # When using AlphaPO transformation, use the transformed rewards - chosen_rewards = self.beta * policy_chosen_rewards.to(self.accelerator.device).detach() - rejected_rewards = self.beta * policy_rejected_rewards.to(self.accelerator.device).detach() - else: - # Standard log probability rewards - chosen_rewards = self.beta * (policy_chosen_logps.to(self.accelerator.device)).detach() - rejected_rewards = self.beta * (policy_rejected_logps.to(self.accelerator.device)).detach() - - return losses, chosen_rewards, rejected_rewards - - @staticmethod - def get_batch_logps( - logits: torch.FloatTensor, - labels: torch.LongTensor, - average_log_prob: bool = False, - label_pad_token_id: int = -100, - is_encoder_decoder: bool = False, - ) -> torch.FloatTensor: - """Compute the log probabilities of the given labels under the given logits. - - Args: - logits: Logits of the model (unnormalized). Shape: (batch_size, sequence_length, vocab_size) - labels: - Labels for which to compute the log probabilities. Label tokens with a value of label_pad_token_id are - ignored. Shape: (batch_size, sequence_length) - average_log_prob: - If True, return the average log probability per (non-masked) token. Otherwise, return the sum of the - log probabilities of the (non-masked) tokens. - label_pad_token_id: The label pad token id. - is_encoder_decoder: Whether the model is an encoder-decoder model. - - Returns: - A tensor of shape (batch_size,) containing the average/sum log probabilities of the given labels under the - given logits. - """ - if logits.shape[:-1] != labels.shape: - raise ValueError("Logits (batch and sequence length dim) and labels must have the same shape.") - - if not is_encoder_decoder: - labels = labels[:, 1:].clone() - logits = logits[:, :-1, :] - loss_mask = labels != label_pad_token_id - - # dummy token; we'll ignore the losses on these tokens later - labels[labels == label_pad_token_id] = 0 - - per_token_logps = selective_log_softmax(logits, labels) - - if average_log_prob: - return (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1) - else: - return (per_token_logps * loss_mask).sum(-1) - - def concatenated_forward( - self, model: nn.Module, batch: dict[str, list | torch.LongTensor] - ) -> tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: - """Run the given model on the given batch of inputs, concatenating the chosen and rejected inputs together. - - We do this to avoid doing two forward passes, because it's faster for FSDP. - """ - concatenated_batch = self.concatenated_inputs( - batch, - is_encoder_decoder=self.is_encoder_decoder, - label_pad_token_id=self.label_pad_token_id, - padding_value=self.padding_value, - device=self.accelerator.device, - ) - len_chosen = batch["chosen_labels"].shape[0] - - model_kwargs = ( - { - "decoder_input_ids": self._shift_right(concatenated_batch["concatenated_labels"]), - } - if self.is_encoder_decoder - else {} - ) - - if self.aux_loss_enabled: - model_kwargs["output_router_logits"] = True - - outputs = model( - concatenated_batch["concatenated_input_ids"], - attention_mask=concatenated_batch["concatenated_attention_mask"], - use_cache=False, - **model_kwargs, - ) - all_logits = outputs.logits - - def cross_entropy_loss(logits, labels): - if not self.is_encoder_decoder: - # Shift so that tokens < n predict n - logits = logits[..., :-1, :].contiguous() - labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = nn.CrossEntropyLoss() - logits = logits.view(-1, logits.shape[-1]) - labels = labels.view(-1) - # Enable model parallelism - labels = labels.to(logits.device) - loss = loss_fct(logits, labels) - return loss - - labels = concatenated_batch["concatenated_labels"].clone() - - if self.cpo_alpha == 0: - nll_loss = torch.tensor(0.0).to(self.accelerator.device) - else: - nll_loss = cross_entropy_loss(all_logits[:len_chosen], labels[:len_chosen]) - - all_logps = self.get_batch_logps( - all_logits, - concatenated_batch["concatenated_labels"], - average_log_prob=self.loss_type in ["ipo", "simpo"], - is_encoder_decoder=self.is_encoder_decoder, - label_pad_token_id=self.label_pad_token_id, - ) - - chosen_logps = all_logps[:len_chosen] - rejected_logps = all_logps[len_chosen:] - - chosen_logits = all_logits[:len_chosen] - rejected_logits = all_logits[len_chosen:] - - if self.aux_loss_enabled: - return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss, outputs.aux_loss) - - return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss) - - def get_batch_loss_metrics( - self, - model, - batch: dict[str, list | torch.LongTensor], - train_eval: Literal["train", "eval"] = "train", - ): - """Compute the CPO loss and other metrics for the given batch of inputs for train or test.""" - metrics = {} - - forward_output = self.concatenated_forward(model, batch) - ( - policy_chosen_logps, - policy_rejected_logps, - policy_chosen_logits, - policy_rejected_logits, - policy_nll_loss, - ) = forward_output[:5] - if self.aux_loss_enabled: - aux_loss = forward_output[5] - - losses, chosen_rewards, rejected_rewards = self.cpo_loss( - policy_chosen_logps, - policy_rejected_logps, - ) - - loss = losses.mean() + self.cpo_alpha * policy_nll_loss - reward_accuracies = (chosen_rewards > rejected_rewards).float() - - prefix = "eval_" if train_eval == "eval" else "" - metrics[f"{prefix}rewards/chosen"] = self.accelerator.gather_for_metrics(chosen_rewards).mean().item() - metrics[f"{prefix}rewards/rejected"] = self.accelerator.gather_for_metrics(rejected_rewards).mean().item() - metrics[f"{prefix}rewards/accuracies"] = self.accelerator.gather_for_metrics(reward_accuracies).mean().item() - metrics[f"{prefix}rewards/margins"] = ( - self.accelerator.gather_for_metrics(chosen_rewards - rejected_rewards).mean().item() - ) - metrics[f"{prefix}logps/rejected"] = ( - self.accelerator.gather_for_metrics(policy_rejected_logps).detach().mean().item() - ) - metrics[f"{prefix}logps/chosen"] = ( - self.accelerator.gather_for_metrics(policy_chosen_logps).detach().mean().item() - ) - metrics[f"{prefix}logits/rejected"] = ( - self.accelerator.gather_for_metrics(policy_rejected_logits.detach().mean()).mean().item() - ) - metrics[f"{prefix}logits/chosen"] = ( - self.accelerator.gather_for_metrics(policy_chosen_logits.detach().mean()).mean().item() - ) - metrics[f"{prefix}nll_loss"] = self.accelerator.gather_for_metrics(policy_nll_loss).detach().mean().item() - - if self.aux_loss_enabled: - loss += self.aux_loss_coef * aux_loss - - return loss, metrics - - def compute_loss( - self, - model: PreTrainedModel | nn.Module, - inputs: dict[str, torch.Tensor | Any], - return_outputs=False, - num_items_in_batch=None, - ) -> torch.Tensor | tuple[torch.Tensor, dict[str, torch.Tensor]]: - compute_loss_context_manager = ( - autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() - ) - - with compute_loss_context_manager: - loss, metrics = self.get_batch_loss_metrics(model, inputs, train_eval="train") - - # force log the metrics - self.store_metrics(metrics, train_eval="train") - - if return_outputs: - return (loss, metrics) - return loss - - def generate_from_model(self, model, batch: dict[str, torch.LongTensor]) -> str: - """Generate samples from the model and reference model for the given batch of inputs.""" - - # If one uses `generate_during_eval` with peft + bf16, we need to explicitly call generate with - # the torch amp context manager as some hidden states are silently casted to full precision. - generate_context_manager = ( - autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() - ) - - with generate_context_manager: - policy_output = model.generate( - input_ids=batch["prompt_input_ids"], - attention_mask=batch["prompt_attention_mask"], - max_length=self.max_length, - do_sample=True, - pad_token_id=self.processing_class.pad_token_id, - ) - - policy_output = pad_to_length(policy_output, self.max_length, self.processing_class.pad_token_id) - policy_output_decoded = self.processing_class.batch_decode(policy_output, skip_special_tokens=True) - - return policy_output_decoded - - def prediction_step( - self, - model: PreTrainedModel | nn.Module, - inputs: dict[str, torch.Tensor | Any], - prediction_loss_only: bool, - ignore_keys: list[str] | None = None, - ): - if ignore_keys is None: - if hasattr(model, "config"): - ignore_keys = getattr(model.config, "keys_to_ignore_at_inference", []) - else: - ignore_keys = [] - - prediction_context_manager = ( - autocast(self.accelerator.device.type) if self._peft_has_been_casted_to_bf16 else nullcontext() - ) - - with torch.no_grad(), prediction_context_manager: - loss, metrics = self.get_batch_loss_metrics(model, inputs, train_eval="eval") - - # force log the metrics - self.store_metrics(metrics, train_eval="eval") - - if prediction_loss_only: - return (loss.detach(), None, None) - - # logits for the chosen and rejected samples from model - logits_dict = { - "eval_logits/chosen": metrics["eval_logits/chosen"], - "eval_logits/rejected": metrics["eval_logits/rejected"], - } - logits = [v for k, v in logits_dict.items() if k not in ignore_keys] - logits = torch.tensor(logits, device=self.accelerator.device) - labels = torch.zeros(logits.shape[0], device=self.accelerator.device) - - return (loss.detach(), logits, labels) - - def store_metrics(self, metrics: dict[str, float], train_eval: Literal["train", "eval"] = "train") -> None: - for key, value in metrics.items(): - self._stored_metrics[train_eval][key].append(value) - - def evaluation_loop( - self, - dataloader: DataLoader, - description: str, - prediction_loss_only: bool | None = None, - ignore_keys: list[str] | None = None, - metric_key_prefix: str = "eval", - ) -> EvalLoopOutput: - """ - Overriding built-in evaluation loop to store metrics for each batch. Prediction/evaluation loop, shared by - `Trainer.evaluate()` and `Trainer.predict()`. - - Works both with or without labels. - """ - - # Sample and save to game log if requested (for one batch to save time) - if self.generate_during_eval: - # Generate random indices within the range of the total number of samples - num_samples = len(dataloader.dataset) - random_indices = random.sample(range(num_samples), k=self.args.eval_batch_size) - - # Use dataloader.dataset.select to get the random batch without iterating over the DataLoader - random_batch_dataset = dataloader.dataset.select(random_indices) - random_batch = self.data_collator(random_batch_dataset) - random_batch = self._prepare_inputs(random_batch) - - policy_output_decoded = self.generate_from_model(self.model, random_batch) - - table = pd.DataFrame( - columns=["Prompt", "Policy"], - data=[ - [prompt, pol[len(prompt) :]] - for prompt, pol in zip(random_batch["prompt"], policy_output_decoded, strict=True) - ], - ) - if "wandb" in self.args.report_to: - wandb.log({"game_log": wandb.Table(data=table)}) - - if "comet_ml" in self.args.report_to: - log_table_to_comet_experiment( - name="game_log.csv", - table=table, - ) - - # Base evaluation - initial_output = super().evaluation_loop( - dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix - ) - - return initial_output - - def log(self, logs: dict[str, float], start_time: float | None = None) -> None: - """ - Log `logs` on the various objects watching training, including stored metrics. - - Args: - logs (`dict[str, float]`): - The values to log. - start_time (`float`, *optional*): - Start time of the training. - """ - # logs either has 'loss' or 'eval_loss' - train_eval = "train" if "loss" in logs else "eval" - # Add averaged stored metrics to logs - for key, metrics in self._stored_metrics[train_eval].items(): - logs[key] = torch.tensor(metrics).mean().item() - del self._stored_metrics[train_eval] - return super().log(logs, start_time) - - def _shift_right(self, input_ids): - if self.decoder_start_token_id is None: - raise ValueError( - "model.config.decoder_start_token_id has to be defined. It is usually set to the pad_token_id." - ) - - # shift inputs to the right - if is_torch_fx_proxy(input_ids): - # Item assignment is not supported natively for proxies. - shifted_input_ids = torch.full(input_ids.shape[:-1] + (1,), self.decoder_start_token_id) - shifted_input_ids = torch.cat([shifted_input_ids, input_ids[..., :-1]], dim=-1) - else: - shifted_input_ids = input_ids.new_zeros(input_ids.shape) - shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() - shifted_input_ids[..., 0] = self.decoder_start_token_id - - if self.pad_token_id is None: - raise ValueError("model.config.pad_token_id has to be defined.") - # replace possible -100 values in labels by `pad_token_id` - shifted_input_ids.masked_fill_(shifted_input_ids == -100, self.pad_token_id) - - return shifted_input_ids - - # Ensure the model card is saved along with the checkpoint - def _save_checkpoint(self, model, trial): - if self.args.hub_model_id is None: - model_name = Path(self.args.output_dir).name - else: - model_name = self.args.hub_model_id.split("/")[-1] - self.create_model_card(model_name=model_name) - super()._save_checkpoint(model, trial) From 294f5fe78b28e9deae63d6219fe22e65a4571e40 Mon Sep 17 00:00:00 2001 From: Behrooz Date: Wed, 5 Nov 2025 18:53:43 -0800 Subject: [PATCH 2/8] Fix ruff linting errors for CPOTrainer migration - Remove unused imports (os, warnings) from experimental/cpo/cpo_trainer.py - Reorder config import after utils imports - Split combined import into separate lines in trainer/cpo_trainer.py - Add missing torch import for type hints in deprecation stub All ruff checks now pass. --- trl/experimental/cpo/cpo_trainer.py | 4 +--- trl/trainer/cpo_trainer.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trl/experimental/cpo/cpo_trainer.py b/trl/experimental/cpo/cpo_trainer.py index 30974f60888..d63ff51cad0 100644 --- a/trl/experimental/cpo/cpo_trainer.py +++ b/trl/experimental/cpo/cpo_trainer.py @@ -13,10 +13,8 @@ # limitations under the License. import inspect -import os import random import textwrap -import warnings from collections import defaultdict from collections.abc import Callable from contextlib import nullcontext @@ -49,7 +47,6 @@ from ...data_utils import maybe_apply_chat_template, maybe_extract_prompt from ...trainer.base_trainer import BaseTrainer -from .cpo_config import CPOConfig from ...trainer.utils import ( DPODataCollatorWithPadding, add_bos_token_if_needed, @@ -60,6 +57,7 @@ peft_module_casting_to_bf16, selective_log_softmax, ) +from .cpo_config import CPOConfig if is_peft_available(): diff --git a/trl/trainer/cpo_trainer.py b/trl/trainer/cpo_trainer.py index 3ce934d686b..b971bdeb41e 100644 --- a/trl/trainer/cpo_trainer.py +++ b/trl/trainer/cpo_trainer.py @@ -15,6 +15,7 @@ import warnings from collections.abc import Callable +import torch import torch.nn as nn from datasets import Dataset from transformers import ( @@ -28,7 +29,8 @@ from transformers.trainer_callback import TrainerCallback from transformers.trainer_utils import EvalLoopOutput -from ..experimental.cpo import CPOConfig, CPOTrainer as _CPOTrainer +from ..experimental.cpo import CPOConfig +from ..experimental.cpo import CPOTrainer as _CPOTrainer class CPOTrainer(_CPOTrainer): From 69811b5627cb5211cf1c493516612ff0c601b53e Mon Sep 17 00:00:00 2001 From: Behrooz Date: Mon, 10 Nov 2025 15:31:21 -0800 Subject: [PATCH 3/8] docs: update CPO references to experimental module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update dataset_formats.md: CPOTrainer → experimental.cpo.CPOTrainer - Update example_overview.md: CPOTrainer → experimental.cpo.CPOTrainer - Update index.md: Add 🧪 emoji to experimental.cpo.CPOTrainer - Update cpo_trainer.md: all trainer references to experimental path - Move test file to tests/experimental/test_cpo_trainer.py - Update test imports to use parent directory reference Follows pattern from XPO PR #4485 --- docs/source/cpo_trainer.md | 10 +++++----- docs/source/dataset_formats.md | 2 +- docs/source/example_overview.md | 2 +- docs/source/index.md | 2 +- tests/{ => experimental}/test_cpo_trainer.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename tests/{ => experimental}/test_cpo_trainer.py (99%) diff --git a/docs/source/cpo_trainer.md b/docs/source/cpo_trainer.md index c2c0f7746b1..adae2f5f208 100644 --- a/docs/source/cpo_trainer.md +++ b/docs/source/cpo_trainer.md @@ -44,7 +44,7 @@ accelerate launch train_cpo.py ## Expected dataset type -CPO requires a [preference dataset](dataset_formats#preference). The [`CPOTrainer`] supports both [conversational](dataset_formats#conversational) and [standard](dataset_formats#standard) dataset formats. When provided with a conversational dataset, the trainer will automatically apply the chat template to the dataset. +CPO requires a [preference dataset](dataset_formats#preference). The [`experimental.cpo.CPOTrainer`] supports both [conversational](dataset_formats#conversational) and [standard](dataset_formats#standard) dataset formats. When provided with a conversational dataset, the trainer will automatically apply the chat template to the dataset. ## Example script @@ -80,7 +80,7 @@ The abstract from the paper is the following: > Direct Preference Optimization (DPO) is a widely used offline preference optimization algorithm that reparameterizes reward functions in reinforcement learning from human feedback (RLHF) to enhance simplicity and training stability. In this work, we propose SimPO, a simpler yet more effective approach. The effectiveness of SimPO is attributed to a key design: using the average log probability of a sequence as the implicit reward. This reward formulation better aligns with model generation and eliminates the need for a reference model, making it more compute and memory efficient. Additionally, we introduce a target reward margin to the Bradley-Terry objective to encourage a larger margin between the winning and losing responses, further enhancing the algorithm's performance. We compare SimPO to DPO and its latest variants across various state-of-the-art training setups, including both base and instruction-tuned models like Mistral and Llama3. We evaluated on extensive instruction-following benchmarks, including AlpacaEval 2, MT-Bench, and the recent challenging Arena-Hard benchmark. Our results demonstrate that SimPO consistently and significantly outperforms existing approaches without substantially increasing response length. Specifically, SimPO outperforms DPO by up to 6.4 points on AlpacaEval 2 and by up to 7.5 points on Arena-Hard. Our top-performing model, built on Llama3-8B-Instruct, achieves a remarkable 44.7 length-controlled win rate on AlpacaEval 2 -- surpassing Claude 3 Opus on the leaderboard, and a 33.8 win rate on Arena-Hard -- making it the strongest 8B open-source model. -The SimPO loss is integrated in the [`CPOTrainer`], as it's an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, just turn on `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and set the `simpo_gamma` to a recommended value. +The SimPO loss is integrated in the [`experimental.cpo.CPOTrainer`], as it's an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, just turn on `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and set the `simpo_gamma` to a recommended value. ### CPO-SimPO @@ -88,7 +88,7 @@ We also offer the combined use of CPO and SimPO, which enables more stable train ### AlphaPO -The [AlphaPO -- Reward shape matters for LLM alignment](https://huggingface.co/papers/2501.03884) (AlphaPO) method by Aman Gupta, Shao Tang, Qingquan Song, Sirou Zhu, [Jiwoo Hong](https://huggingface.co/JW17), Ankan Saha, Viral Gupta, Noah Lee, Eunki Kim, Jason Zhu, Natesh Pillai, and S. Sathiya Keerthi is also implemented in the [`CPOTrainer`]. AlphaPO is an alternative method that applies a transformation to the reward function shape in the context of SimPO loss. The abstract from the paper is the following: +The [AlphaPO -- Reward shape matters for LLM alignment](https://huggingface.co/papers/2501.03884) (AlphaPO) method by Aman Gupta, Shao Tang, Qingquan Song, Sirou Zhu, [Jiwoo Hong](https://huggingface.co/JW17), Ankan Saha, Viral Gupta, Noah Lee, Eunki Kim, Jason Zhu, Natesh Pillai, and S. Sathiya Keerthi is also implemented in the [`experimental.cpo.CPOTrainer`]. AlphaPO is an alternative method that applies a transformation to the reward function shape in the context of SimPO loss. The abstract from the paper is the following: > Reinforcement Learning with Human Feedback (RLHF) and its variants have made huge strides toward the effective alignment of large language models (LLMs) to follow instructions and reflect human values. More recently, Direct Alignment Algorithms (DAAs) have emerged in which the reward modeling stage of RLHF is skipped by characterizing the reward directly as a function of the policy being learned. Some popular examples of DAAs include Direct Preference Optimization (DPO) and Simple Preference Optimization (SimPO). These methods often suffer from likelihood displacement, a phenomenon by which the probabilities of preferred responses are often reduced undesirably. In this paper, we argue that, for DAAs the reward (function) shape matters. We introduce AlphaPO, a new DAA method that leverages an α-parameter to help change the shape of the reward function beyond the standard log reward. AlphaPO helps maintain fine-grained control over likelihood displacement and overoptimization. Compared to SimPO, one of the best performing DAAs, AlphaPO leads to about 7% to 10% relative improvement in alignment performance for the instruct versions of Mistral-7B and Llama3-8B while achieving 15% to 50% relative improvement over DPO on the same models. The analysis and results presented highlight the importance of the reward shape and how one can systematically change it to affect training dynamics, as well as improve alignment performance. @@ -103,8 +103,8 @@ The CPO algorithm supports several loss functions. The loss function can be set | `"sigmoid"` (default) | Given the preference data, we can fit a binary classifier according to the Bradley-Terry model, and in fact, the [DPO](https://huggingface.co/papers/2305.18290) authors propose the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression. | | `"hinge"` | The [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. In this case, the `beta` is the reciprocal of the margin. | | `"ipo"` | The [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the DPO algorithms and identify an issue with overfitting and propose an alternative loss. In this case, the `beta` is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair, and thus the smaller the `beta`, the larger this gap is. As per the paper, the loss is averaged over log-likelihoods of the completion (unlike DPO, which is summed only). | -| `"simpo"` | The [SimPO](https://huggingface.co/papers/2405.14734) method is also implemented in the [`CPOTrainer`]. SimPO is an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, simply set `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and `simpo_gamma` to a recommended value. | -| `"alphapo"` | The [AlphaPO](https://huggingface.co/papers/2501.03884) method is also implemented in the [`CPOTrainer`]. This is syntactic sugar that automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`. AlphaPO applies a transformation to the reward function shape in the context of SimPO loss when the `alpha` parameter is non-zero. | +| `"simpo"` | The [SimPO](https://huggingface.co/papers/2405.14734) method is also implemented in the [`experimental.cpo.CPOTrainer`]. SimPO is an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, simply set `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and `simpo_gamma` to a recommended value. | +| `"alphapo"` | The [AlphaPO](https://huggingface.co/papers/2501.03884) method is also implemented in the [`experimental.cpo.CPOTrainer`]. This is syntactic sugar that automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`. AlphaPO applies a transformation to the reward function shape in the context of SimPO loss when the `alpha` parameter is non-zero. | ### For Mixture of Experts Models: Enabling the auxiliary loss diff --git a/docs/source/dataset_formats.md b/docs/source/dataset_formats.md index e9406df731b..6e371038457 100644 --- a/docs/source/dataset_formats.md +++ b/docs/source/dataset_formats.md @@ -388,7 +388,7 @@ Choosing the right dataset type depends on the task you are working on and the s | Trainer | Expected dataset type | | --- | --- | | [`experimental.bco.BCOTrainer`] | [Unpaired preference](#unpaired-preference) or [Preference (explicit prompt recommended)](#preference) | -| [`CPOTrainer`] | [Preference (explicit prompt recommended)](#preference) | +| [`experimental.cpo.CPOTrainer`] | [Preference (explicit prompt recommended)](#preference) | | [`DPOTrainer`] | [Preference (explicit prompt recommended)](#preference) | | [`GKDTrainer`] | [Prompt-completion](#prompt-completion) | | [`GRPOTrainer`] | [Prompt-only](#prompt-only) | diff --git a/docs/source/example_overview.md b/docs/source/example_overview.md index d3905768df5..09ed2f2147e 100644 --- a/docs/source/example_overview.md +++ b/docs/source/example_overview.md @@ -40,7 +40,7 @@ Scripts are maintained in the [`trl/scripts`](https://github.com/huggingface/trl File | Description | | --- | --- | | [`examples/scripts/bco.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/bco.py) | This script shows how to use the [`KTOTrainer`] with the BCO loss to fine-tune a model to increase instruction-following, truthfulness, honesty, and helpfulness using the [openbmb/UltraFeedback](https://huggingface.co/datasets/openbmb/UltraFeedback) dataset. | -| [`examples/scripts/cpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/cpo.py) | This script shows how to use the [`CPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset. | +| [`examples/scripts/cpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/cpo.py) | This script shows how to use the [`experimental.cpo.CPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset. | | [`trl/scripts/dpo.py`](https://github.com/huggingface/trl/blob/main/trl/scripts/dpo.py) | This script shows how to use the [`DPOTrainer`] to fine-tune a model. | | [`examples/scripts/dpo_vlm.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo_vlm.py) | This script shows how to use the [`DPOTrainer`] to fine-tune a Vision Language Model to reduce hallucinations using the [openbmb/RLAIF-V-Dataset](https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset) dataset. | | [`examples/scripts/evals/judge_tldr.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/evals/judge_tldr.py) | This script shows how to use [`HfPairwiseJudge`] or [`experimental.judges.OpenAIPairwiseJudge`] to judge model generations. | diff --git a/docs/source/index.md b/docs/source/index.md index 06aca201b17..e121691f22b 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -43,7 +43,7 @@ Below is the current list of TRL trainers, organized by method type (⚡️ = vL - [`DPOTrainer`] - [`ORPOTrainer`] - [`experimental.bco.BCOTrainer`] 🧪 -- [`CPOTrainer`] +- [`experimental.cpo.CPOTrainer`] 🧪 - [`KTOTrainer`] ### Knowledge distillation diff --git a/tests/test_cpo_trainer.py b/tests/experimental/test_cpo_trainer.py similarity index 99% rename from tests/test_cpo_trainer.py rename to tests/experimental/test_cpo_trainer.py index 7aecc54df5d..b699c58bbc3 100644 --- a/tests/test_cpo_trainer.py +++ b/tests/experimental/test_cpo_trainer.py @@ -19,7 +19,7 @@ from trl.experimental.cpo import CPOConfig, CPOTrainer -from .testing_utils import TrlTestCase, require_peft +from ..testing_utils import TrlTestCase, require_peft class TestCPOTrainer(TrlTestCase): From 864c1b974f0e1bed569b6a1cfa1d57cba11c9d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 12 Nov 2025 23:47:44 +0000 Subject: [PATCH 4/8] consistency --- docs/source/cpo_trainer.md | 14 ++-- docs/source/index.md | 4 +- docs/source/paper_index.md | 2 +- tests/experimental/test_trainers_args.py | 42 ++++++++++ trl/experimental/cpo/__init__.py | 3 + trl/experimental/cpo/cpo_config.py | 2 +- trl/experimental/cpo/cpo_trainer.py | 2 +- trl/trainer/cpo_config.py | 12 +-- trl/trainer/cpo_trainer.py | 98 ++---------------------- 9 files changed, 63 insertions(+), 116 deletions(-) diff --git a/docs/source/cpo_trainer.md b/docs/source/cpo_trainer.md index adae2f5f208..e1ff2a198a4 100644 --- a/docs/source/cpo_trainer.md +++ b/docs/source/cpo_trainer.md @@ -80,11 +80,11 @@ The abstract from the paper is the following: > Direct Preference Optimization (DPO) is a widely used offline preference optimization algorithm that reparameterizes reward functions in reinforcement learning from human feedback (RLHF) to enhance simplicity and training stability. In this work, we propose SimPO, a simpler yet more effective approach. The effectiveness of SimPO is attributed to a key design: using the average log probability of a sequence as the implicit reward. This reward formulation better aligns with model generation and eliminates the need for a reference model, making it more compute and memory efficient. Additionally, we introduce a target reward margin to the Bradley-Terry objective to encourage a larger margin between the winning and losing responses, further enhancing the algorithm's performance. We compare SimPO to DPO and its latest variants across various state-of-the-art training setups, including both base and instruction-tuned models like Mistral and Llama3. We evaluated on extensive instruction-following benchmarks, including AlpacaEval 2, MT-Bench, and the recent challenging Arena-Hard benchmark. Our results demonstrate that SimPO consistently and significantly outperforms existing approaches without substantially increasing response length. Specifically, SimPO outperforms DPO by up to 6.4 points on AlpacaEval 2 and by up to 7.5 points on Arena-Hard. Our top-performing model, built on Llama3-8B-Instruct, achieves a remarkable 44.7 length-controlled win rate on AlpacaEval 2 -- surpassing Claude 3 Opus on the leaderboard, and a 33.8 win rate on Arena-Hard -- making it the strongest 8B open-source model. -The SimPO loss is integrated in the [`experimental.cpo.CPOTrainer`], as it's an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, just turn on `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and set the `simpo_gamma` to a recommended value. +The SimPO loss is integrated in the [`experimental.cpo.CPOTrainer`], as it's an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, just turn on `loss_type="simpo"` and `cpo_alpha=0.0` in the [`experimental.cpo.CPOConfig`] and set the `simpo_gamma` to a recommended value. ### CPO-SimPO -We also offer the combined use of CPO and SimPO, which enables more stable training and improved performance. Learn more details at [CPO-SimPO GitHub](https://github.com/fe1ixxu/CPO_SIMPO). To use this method, simply enable SimPO by setting `loss_type="simpo"` and a non-zero `cpo_alpha` in the [`CPOConfig`]. +We also offer the combined use of CPO and SimPO, which enables more stable training and improved performance. Learn more details at [CPO-SimPO GitHub](https://github.com/fe1ixxu/CPO_SIMPO). To use this method, simply enable SimPO by setting `loss_type="simpo"` and a non-zero `cpo_alpha` in the [`experimental.cpo.CPOConfig`]. ### AlphaPO @@ -92,18 +92,18 @@ The [AlphaPO -- Reward shape matters for LLM alignment](https://huggingface.co/p > Reinforcement Learning with Human Feedback (RLHF) and its variants have made huge strides toward the effective alignment of large language models (LLMs) to follow instructions and reflect human values. More recently, Direct Alignment Algorithms (DAAs) have emerged in which the reward modeling stage of RLHF is skipped by characterizing the reward directly as a function of the policy being learned. Some popular examples of DAAs include Direct Preference Optimization (DPO) and Simple Preference Optimization (SimPO). These methods often suffer from likelihood displacement, a phenomenon by which the probabilities of preferred responses are often reduced undesirably. In this paper, we argue that, for DAAs the reward (function) shape matters. We introduce AlphaPO, a new DAA method that leverages an α-parameter to help change the shape of the reward function beyond the standard log reward. AlphaPO helps maintain fine-grained control over likelihood displacement and overoptimization. Compared to SimPO, one of the best performing DAAs, AlphaPO leads to about 7% to 10% relative improvement in alignment performance for the instruct versions of Mistral-7B and Llama3-8B while achieving 15% to 50% relative improvement over DPO on the same models. The analysis and results presented highlight the importance of the reward shape and how one can systematically change it to affect training dynamics, as well as improve alignment performance. -To use this loss as described in the paper, we can set the `loss_type="alphapo"` which automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`, together with `alpha` and `simpo_gamma` to recommended values in the [`CPOConfig`]. Alternatively, you can manually set `loss_type="simpo"`, `cpo_alpha=0.0`, together with `alpha` and `simpo_gamma` to recommended values. Other variants of this method are also possible, such as setting `loss_type="ipo"` and `alpha` to any non-zero value. +To use this loss as described in the paper, we can set the `loss_type="alphapo"` which automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`, together with `alpha` and `simpo_gamma` to recommended values in the [`experimental.cpo.CPOConfig`]. Alternatively, you can manually set `loss_type="simpo"`, `cpo_alpha=0.0`, together with `alpha` and `simpo_gamma` to recommended values. Other variants of this method are also possible, such as setting `loss_type="ipo"` and `alpha` to any non-zero value. ## Loss functions -The CPO algorithm supports several loss functions. The loss function can be set using the `loss_type` parameter in the [`CPOConfig`]. The following loss functions are supported: +The CPO algorithm supports several loss functions. The loss function can be set using the `loss_type` parameter in the [`experimental.cpo.CPOConfig`]. The following loss functions are supported: | `loss_type=` | Description | | --- | --- | | `"sigmoid"` (default) | Given the preference data, we can fit a binary classifier according to the Bradley-Terry model, and in fact, the [DPO](https://huggingface.co/papers/2305.18290) authors propose the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression. | | `"hinge"` | The [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. In this case, the `beta` is the reciprocal of the margin. | | `"ipo"` | The [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the DPO algorithms and identify an issue with overfitting and propose an alternative loss. In this case, the `beta` is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair, and thus the smaller the `beta`, the larger this gap is. As per the paper, the loss is averaged over log-likelihoods of the completion (unlike DPO, which is summed only). | -| `"simpo"` | The [SimPO](https://huggingface.co/papers/2405.14734) method is also implemented in the [`experimental.cpo.CPOTrainer`]. SimPO is an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, simply set `loss_type="simpo"` and `cpo_alpha=0.0` in the [`CPOConfig`] and `simpo_gamma` to a recommended value. | +| `"simpo"` | The [SimPO](https://huggingface.co/papers/2405.14734) method is also implemented in the [`experimental.cpo.CPOTrainer`]. SimPO is an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, simply set `loss_type="simpo"` and `cpo_alpha=0.0` in the [`experimental.cpo.CPOConfig`] and `simpo_gamma` to a recommended value. | | `"alphapo"` | The [AlphaPO](https://huggingface.co/papers/2501.03884) method is also implemented in the [`experimental.cpo.CPOTrainer`]. This is syntactic sugar that automatically sets `loss_type="simpo"` and `cpo_alpha=0.0`. AlphaPO applies a transformation to the reward function shape in the context of SimPO loss when the `alpha` parameter is non-zero. | ### For Mixture of Experts Models: Enabling the auxiliary loss @@ -116,11 +116,11 @@ To scale how much the auxiliary loss contributes to the total loss, use the hype ## CPOTrainer -[[autodoc]] CPOTrainer +[[autodoc]] experimental.cpo.CPOTrainer - train - save_model - push_to_hub ## CPOConfig -[[autodoc]] CPOConfig +[[autodoc]] experimental.cpo.CPOConfig diff --git a/docs/source/index.md b/docs/source/index.md index e121691f22b..4fcd1b68aaf 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -26,8 +26,8 @@ Below is the current list of TRL trainers, organized by method type (⚡️ = vL - [`RLOOTrainer`] ⚡️ - [`OnlineDPOTrainer`] ⚡️ - [`NashMDTrainer`] ⚡️ -- [`experimental.xpo.XPOTrainer`] 🧪 ⚡️ - [`PPOTrainer`] +- [`experimental.xpo.XPOTrainer`] 🧪 ⚡️ ### Reward modeling @@ -42,9 +42,9 @@ Below is the current list of TRL trainers, organized by method type (⚡️ = vL - [`SFTTrainer`] - [`DPOTrainer`] - [`ORPOTrainer`] +- [`KTOTrainer`] - [`experimental.bco.BCOTrainer`] 🧪 - [`experimental.cpo.CPOTrainer`] 🧪 -- [`KTOTrainer`] ### Knowledge distillation diff --git a/docs/source/paper_index.md b/docs/source/paper_index.md index b94808112e8..2921588b6b5 100644 --- a/docs/source/paper_index.md +++ b/docs/source/paper_index.md @@ -556,7 +556,7 @@ training_args = RLOOConfig( ## Contrastive Preference Optimization -Papers relating to the [`CPOTrainer`] +Papers relating to the [`experimental.cpo.CPOTrainer`] ### AlphaPO -- Reward shape matters for LLM alignment diff --git a/tests/experimental/test_trainers_args.py b/tests/experimental/test_trainers_args.py index 3c1d93cfc46..b9febaebfa2 100644 --- a/tests/experimental/test_trainers_args.py +++ b/tests/experimental/test_trainers_args.py @@ -17,6 +17,7 @@ from transformers import AutoModelForCausalLM, AutoModelForSequenceClassification, AutoTokenizer from trl.experimental.bco import BCOConfig, BCOTrainer +from trl.experimental.cpo import CPOConfig, CPOTrainer from trl.experimental.xpo import XPOConfig, XPOTrainer from ..testing_utils import TrlTestCase, require_sklearn @@ -71,6 +72,47 @@ def test_bco(self): assert trainer.args.min_density_ratio == 0.2 assert trainer.args.max_density_ratio == 20.0 + def test_cpo(self): + model_id = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5" + tokenizer = AutoTokenizer.from_pretrained(model_id) + dataset = load_dataset("trl-internal-testing/zen", "standard_preference", split="train") + training_args = CPOConfig( + self.tmp_dir, + max_length=256, + max_prompt_length=64, + max_completion_length=64, + beta=0.5, + label_smoothing=0.5, + loss_type="hinge", + disable_dropout=False, + cpo_alpha=0.5, + simpo_gamma=0.2, + label_pad_token_id=-99, + padding_value=-99, + truncation_mode="keep_start", + # generate_during_eval=True, # ignore this one, it requires wandb + is_encoder_decoder=True, + model_init_kwargs={"trust_remote_code": True}, + dataset_num_proc=4, + ) + trainer = CPOTrainer(model=model_id, args=training_args, train_dataset=dataset, processing_class=tokenizer) + assert trainer.args.max_length == 256 + assert trainer.args.max_prompt_length == 64 + assert trainer.args.max_completion_length == 64 + assert trainer.args.beta == 0.5 + assert trainer.args.label_smoothing == 0.5 + assert trainer.args.loss_type == "hinge" + assert not trainer.args.disable_dropout + assert trainer.args.cpo_alpha == 0.5 + assert trainer.args.simpo_gamma == 0.2 + assert trainer.args.label_pad_token_id == -99 + assert trainer.args.padding_value == -99 + assert trainer.args.truncation_mode == "keep_start" + # self.assertEqual(trainer.args.generate_during_eval, True) + assert trainer.args.is_encoder_decoder + assert trainer.args.model_init_kwargs == {"trust_remote_code": True} + assert trainer.args.dataset_num_proc == 4 + @pytest.mark.parametrize("alpha_list", [False, True]) def test_xpo(self, alpha_list): model_id = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5" diff --git a/trl/experimental/cpo/__init__.py b/trl/experimental/cpo/__init__.py index d542f227fba..e20cbbe3638 100644 --- a/trl/experimental/cpo/__init__.py +++ b/trl/experimental/cpo/__init__.py @@ -14,3 +14,6 @@ from .cpo_config import CPOConfig from .cpo_trainer import CPOTrainer + + +__all__ = ["CPOConfig", "CPOTrainer"] diff --git a/trl/experimental/cpo/cpo_config.py b/trl/experimental/cpo/cpo_config.py index 426b00da153..be9f747a53c 100644 --- a/trl/experimental/cpo/cpo_config.py +++ b/trl/experimental/cpo/cpo_config.py @@ -21,7 +21,7 @@ @dataclass class CPOConfig(TrainingArguments): r""" - Configuration class for the [`CPOTrainer`]. + Configuration class for the [`experimental.cpo.CPOTrainer`]. This class includes only the parameters that are specific to CPO training. For a full list of training arguments, please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may diff --git a/trl/experimental/cpo/cpo_trainer.py b/trl/experimental/cpo/cpo_trainer.py index d63ff51cad0..ceed246e8ac 100644 --- a/trl/experimental/cpo/cpo_trainer.py +++ b/trl/experimental/cpo/cpo_trainer.py @@ -78,7 +78,7 @@ class CPOTrainer(BaseTrainer): Args: model ([`~transformers.PreTrainedModel`]): The model to train, preferably an [`~transformers.AutoModelForSequenceClassification`]. - args ([`CPOConfig`]): + args ([`experimental.cpo.CPOConfig`]): The CPO config arguments to use for training. data_collator ([`~transformers.DataCollator`]): The data collator to use for training. If None is specified, the default data collator diff --git a/trl/trainer/cpo_config.py b/trl/trainer/cpo_config.py index 268224e857a..91df318221d 100644 --- a/trl/trainer/cpo_config.py +++ b/trl/trainer/cpo_config.py @@ -20,20 +20,10 @@ @dataclass class CPOConfig(_CPOConfig): - r""" - Configuration class for the [`CPOTrainer`]. - - This class is now located in `trl.experimental.cpo`. Please update your imports to - `from trl.experimental.cpo import CPOConfig`. The current import path will be removed in TRL 0.29. - For more information, see https://github.com/huggingface/trl/issues/4223. - """ - def __post_init__(self): warnings.warn( "The `CPOConfig` is now located in `trl.experimental`. Please update your imports to " "`from trl.experimental.cpo import CPOConfig`. The current import path will be removed and no longer " - "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223.", - FutureWarning, - stacklevel=2, + "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223." ) super().__post_init__() diff --git a/trl/trainer/cpo_trainer.py b/trl/trainer/cpo_trainer.py index b971bdeb41e..4a89a0aa157 100644 --- a/trl/trainer/cpo_trainer.py +++ b/trl/trainer/cpo_trainer.py @@ -13,105 +13,17 @@ # limitations under the License. import warnings -from collections.abc import Callable +from dataclasses import dataclass -import torch -import torch.nn as nn -from datasets import Dataset -from transformers import ( - BaseImageProcessor, - DataCollator, - FeatureExtractionMixin, - PreTrainedModel, - PreTrainedTokenizerBase, - ProcessorMixin, -) -from transformers.trainer_callback import TrainerCallback -from transformers.trainer_utils import EvalLoopOutput - -from ..experimental.cpo import CPOConfig from ..experimental.cpo import CPOTrainer as _CPOTrainer +@dataclass class CPOTrainer(_CPOTrainer): - r""" - Initialize CPOTrainer. - - This class is now located in `trl.experimental.cpo`. Please update your imports to - `from trl.experimental.cpo import CPOTrainer`. The current import path will be removed in TRL 0.29. - For more information, see https://github.com/huggingface/trl/issues/4223. - - Args: - model ([`~transformers.PreTrainedModel`]): - The model to train, preferably an [`~transformers.AutoModelForSequenceClassification`]. - args ([`CPOConfig`]): - The CPO config arguments to use for training. - data_collator ([`~transformers.DataCollator`]): - The data collator to use for training. If None is specified, the default data collator - ([`DPODataCollatorWithPadding`]) will be used which will pad the sequences to the maximum length of the - sequences in the batch, given a dataset of paired sequences. - train_dataset ([`~datasets.Dataset`]): - The dataset to use for training. - eval_dataset ([`~datasets.Dataset`]): - The dataset to use for evaluation. - processing_class ([`~transformers.PreTrainedTokenizerBase`], [`~transformers.BaseImageProcessor`], [`~transformers.FeatureExtractionMixin`] or [`~transformers.ProcessorMixin`], *optional*): - Processing class used to process the data. If provided, will be used to automatically process the inputs - for the model, and it will be saved along the model to make it easier to rerun an interrupted training or - reuse the fine-tuned model. - model_init (`Callable[[], transformers.PreTrainedModel]`): - The model initializer to use for training. If None is specified, the default model initializer will be - used. - callbacks (`list[transformers.TrainerCallback]`): - The callbacks to use for training. - optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`): - The optimizer and scheduler to use for training. - preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`): - The function to use to preprocess the logits before computing the metrics. - peft_config (`dict`, defaults to `None`): - The PEFT configuration to use for training. If you pass a PEFT configuration, the model will be wrapped in - a PEFT model. - compute_metrics (`Callable[[EvalPrediction], dict]`, *optional*): - The function to use to compute the metrics. Must take a `EvalPrediction` and return a dictionary string to - metric values. - """ - - def __init__( - self, - model: PreTrainedModel | nn.Module | str | None = None, - args: CPOConfig | None = None, - data_collator: DataCollator | None = None, - train_dataset: Dataset | None = None, - eval_dataset: Dataset | dict[str, Dataset] | None = None, - processing_class: PreTrainedTokenizerBase - | BaseImageProcessor - | FeatureExtractionMixin - | ProcessorMixin - | None = None, - model_init: Callable[[], PreTrainedModel] | None = None, - callbacks: list[TrainerCallback] | None = None, - optimizers: tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), - preprocess_logits_for_metrics: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] | None = None, - peft_config: dict | None = None, - compute_metrics: Callable[[EvalLoopOutput], dict] | None = None, - ): + def __init__(self, *args, **kwargs): warnings.warn( "The `CPOTrainer` is now located in `trl.experimental`. Please update your imports to " "`from trl.experimental.cpo import CPOTrainer`. The current import path will be removed and no longer " - "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223.", - FutureWarning, - stacklevel=2, - ) - super().__init__( - model=model, - args=args, - data_collator=data_collator, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - processing_class=processing_class, - model_init=model_init, - callbacks=callbacks, - optimizers=optimizers, - preprocess_logits_for_metrics=preprocess_logits_for_metrics, - peft_config=peft_config, - compute_metrics=compute_metrics, + "supported in TRL 0.29. For more information, see https://github.com/huggingface/trl/issues/4223." ) + super().__init__(*args, **kwargs) From 3eb3d187b8370bb0f217f5be29bbd35da58302ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 12 Nov 2025 23:52:06 +0000 Subject: [PATCH 5/8] finish merge main --- trl/experimental/cpo/cpo_config.py | 10 ++++++++++ trl/experimental/cpo/cpo_trainer.py | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/trl/experimental/cpo/cpo_config.py b/trl/experimental/cpo/cpo_config.py index be9f747a53c..93e7015d17c 100644 --- a/trl/experimental/cpo/cpo_config.py +++ b/trl/experimental/cpo/cpo_config.py @@ -115,6 +115,16 @@ class CPOConfig(TrainingArguments): "`fp16` is not set." }, ) + # Transformers 4.57.0 introduced a bug that caused the dtype of `lr_scheduler_kwargs` to be unparsable. This issue + # was fixed in https://github.com/huggingface/transformers/pull/41322, but the fix has not yet been released. We + # add a temporary workaround here, which can be removed once the fix is available—likely in Transformers 4.57.2. + lr_scheduler_kwargs: dict | str | None = field( + default=None, + metadata={ + "help": "Additional parameters for the lr_scheduler, such as {'num_cycles': 1} for cosine with hard " + "restarts." + }, + ) max_length: int | None = field( default=1024, diff --git a/trl/experimental/cpo/cpo_trainer.py b/trl/experimental/cpo/cpo_trainer.py index ceed246e8ac..cd525b6aac4 100644 --- a/trl/experimental/cpo/cpo_trainer.py +++ b/trl/experimental/cpo/cpo_trainer.py @@ -151,7 +151,7 @@ def __init__( raise ValueError("You passed model_kwargs to the CPOTrainer. But your model is already instantiated.") else: model_init_kwargs = args.model_init_kwargs - dtype = model_init_kwargs.get("dtype") + dtype = model_init_kwargs.get("dtype", "auto") if dtype is not None: # Convert to `torch.dtype` if an str is passed if isinstance(dtype, str) and dtype != "auto": @@ -161,6 +161,7 @@ def __init__( f"Invalid `dtype` passed to the CPOConfig. Expected a string with either `torch.dtype` or 'auto', but got {dtype}." ) model_init_kwargs["dtype"] = dtype + model_init_kwargs["device_map"] = model_init_kwargs.get("device_map", "auto") if isinstance(model, str): model = AutoModelForCausalLM.from_pretrained(model, **model_init_kwargs) From e10008fb86946c88b04545afa889092e5b25d729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 12 Nov 2025 23:54:22 +0000 Subject: [PATCH 6/8] rm train arg test --- tests/test_trainers_args.py | 46 ++----------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/tests/test_trainers_args.py b/tests/test_trainers_args.py index addc5eb1b7a..19679f93d9d 100644 --- a/tests/test_trainers_args.py +++ b/tests/test_trainers_args.py @@ -33,53 +33,11 @@ SFTConfig, SFTTrainer, ) -from trl.experimental.cpo import CPOConfig, CPOTrainer from .testing_utils import TrlTestCase class TestTrainerArg(TrlTestCase): - def test_cpo(self): - model_id = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5" - tokenizer = AutoTokenizer.from_pretrained(model_id) - dataset = load_dataset("trl-internal-testing/zen", "standard_preference", split="train") - training_args = CPOConfig( - self.tmp_dir, - max_length=256, - max_prompt_length=64, - max_completion_length=64, - beta=0.5, - label_smoothing=0.5, - loss_type="hinge", - disable_dropout=False, - cpo_alpha=0.5, - simpo_gamma=0.2, - label_pad_token_id=-99, - padding_value=-99, - truncation_mode="keep_start", - # generate_during_eval=True, # ignore this one, it requires wandb - is_encoder_decoder=True, - model_init_kwargs={"trust_remote_code": True}, - dataset_num_proc=4, - ) - trainer = CPOTrainer(model=model_id, args=training_args, train_dataset=dataset, processing_class=tokenizer) - assert trainer.args.max_length == 256 - assert trainer.args.max_prompt_length == 64 - assert trainer.args.max_completion_length == 64 - assert trainer.args.beta == 0.5 - assert trainer.args.label_smoothing == 0.5 - assert trainer.args.loss_type == "hinge" - assert not trainer.args.disable_dropout - assert trainer.args.cpo_alpha == 0.5 - assert trainer.args.simpo_gamma == 0.2 - assert trainer.args.label_pad_token_id == -99 - assert trainer.args.padding_value == -99 - assert trainer.args.truncation_mode == "keep_start" - # self.assertEqual(trainer.args.generate_during_eval, True) - assert trainer.args.is_encoder_decoder - assert trainer.args.model_init_kwargs == {"trust_remote_code": True, "device_map": "auto", "dtype": "auto"} - assert trainer.args.dataset_num_proc == 4 - def test_dpo(self): model_id = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5" tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -188,8 +146,8 @@ def test_kto(self): # self.assertEqual(trainer.args.generate_during_eval, True) assert trainer.args.is_encoder_decoder assert trainer.args.precompute_ref_log_probs - assert trainer.args.model_init_kwargs == {"trust_remote_code": True, "device_map": "auto", "dtype": "auto"} - assert trainer.args.ref_model_init_kwargs == {"trust_remote_code": True, "device_map": "auto", "dtype": "auto"} + assert trainer.args.model_init_kwargs == {"trust_remote_code": True} + assert trainer.args.ref_model_init_kwargs == {"trust_remote_code": True} assert trainer.args.dataset_num_proc == 4 @pytest.mark.parametrize("mixtures_coef_list", [False, True]) From f79a2e0a131392c20bf4efc47d0699f4bf17a498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 12 Nov 2025 23:55:58 +0000 Subject: [PATCH 7/8] revert --- tests/test_trainers_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_trainers_args.py b/tests/test_trainers_args.py index 19679f93d9d..4884f3247b3 100644 --- a/tests/test_trainers_args.py +++ b/tests/test_trainers_args.py @@ -123,8 +123,8 @@ def test_kto(self): # generate_during_eval=True, # ignore this one, it requires wandb is_encoder_decoder=True, precompute_ref_log_probs=True, - model_init_kwargs={"trust_remote_code": True}, - ref_model_init_kwargs={"trust_remote_code": True}, + model_init_kwargs={"trust_remote_code": True, "device_map": "auto", "dtype": "auto"}, + ref_model_init_kwargs={"trust_remote_code": True, "device_map": "auto", "dtype": "auto"}, dataset_num_proc=4, ) trainer = KTOTrainer( From 689d9498157b6926726937e636679cecfbea9937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 12 Nov 2025 17:34:30 -0700 Subject: [PATCH 8/8] Update test_trainers_args.py --- tests/test_trainers_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_trainers_args.py b/tests/test_trainers_args.py index 4884f3247b3..19679f93d9d 100644 --- a/tests/test_trainers_args.py +++ b/tests/test_trainers_args.py @@ -123,8 +123,8 @@ def test_kto(self): # generate_during_eval=True, # ignore this one, it requires wandb is_encoder_decoder=True, precompute_ref_log_probs=True, - model_init_kwargs={"trust_remote_code": True, "device_map": "auto", "dtype": "auto"}, - ref_model_init_kwargs={"trust_remote_code": True, "device_map": "auto", "dtype": "auto"}, + model_init_kwargs={"trust_remote_code": True}, + ref_model_init_kwargs={"trust_remote_code": True}, dataset_num_proc=4, ) trainer = KTOTrainer(