diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index 838158e4189..ace444e07c3 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -253,6 +253,8 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. +If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing_memory_usage#vllm-sleep-mode). + diff --git a/docs/source/reducing_memory_usage.md b/docs/source/reducing_memory_usage.md index 018738bdf34..1dd2363a5a3 100644 --- a/docs/source/reducing_memory_usage.md +++ b/docs/source/reducing_memory_usage.md @@ -267,3 +267,28 @@ training_args = RLOOConfig(..., ds3_gather_for_generation=False) This adjustment prevents model weights from being gathered, avoiding OOM errors, but it may result in slower generation speeds. + +## vLLM sleep mode + +When using vLLM as the generation backend, you can enable _sleep mode_ to offload vLLM parameters and cache to CPU RAM during the optimization step and reload them back to GPU VRAM when needed for weight synchronization and generation. + + + + +```python +from trl import GRPOConfig + +training_args = GRPOConfig(..., vllm_sleep_enabled=True) +``` + + + + +```python +from trl import RLOOConfig + +training_args = RLOOConfig(..., vllm_sleep_enabled=True) +``` + + + diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index 43c7e966aae..55fabbfe98d 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -140,6 +140,9 @@ class GRPOConfig(TrainingArguments): Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model implementation. + vllm_enable_sleep_mode (`bool`, *optional*, defaults to `False`): + Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken + for weight sync and generation. > Parameters that control the training @@ -416,6 +419,13 @@ class GRPOConfig(TrainingArguments): "model implementation." }, ) + vllm_enable_sleep_mode: bool = field( + default=False, + metadata={ + "help": "Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step " + "and woken for weight sync and generation." + }, + ) vllm_guided_decoding_regex: Optional[str] = field( default=None, metadata={"help": "Regex for vLLM guided decoding. If `None` (default), guided decoding is disabled."}, diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 62626cc54e7..99dd53e7ac6 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -533,10 +533,13 @@ def __init__( distributed_executor_backend="external_launcher", # Feed identical seed for tp groups to ensure sampling results are the same across workers seed=self.accelerator.process_index // self.vllm_tensor_parallel_size, - # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory + # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory max_num_batched_tokens=4096, model_impl=self.args.vllm_model_impl, + enable_sleep_mode=self.args.vllm_enable_sleep_mode, ) + if self.args.vllm_enable_sleep_mode: + self.llm.sleep(level=1) else: raise ValueError(f"vllm_mode must be either 'server' or 'colocate', got '{self.vllm_mode}'.") @@ -1127,6 +1130,11 @@ def _generate_and_score_completions( # Generate completions using either vLLM or regular generation if self.use_vllm: + if self.vllm_mode == "colocate" and self.args.vllm_enable_sleep_mode: + # wake up colocated vLLM instances if needed + torch.cuda.empty_cache() # required to avoid OOM in some cases + self.llm.wake_up() + # First, update the vLLM weights if needed if self.state.global_step != self._last_loaded_step: self._move_model_to_vllm() @@ -1235,6 +1243,9 @@ def _generate_and_score_completions( tp_slice = slice(local_rank_in_group * orig_size, (local_rank_in_group + 1) * orig_size) completion_ids = completion_ids[tp_slice] + if self.args.vllm_enable_sleep_mode: + self.llm.sleep(level=1) + # Pad the completions, and concatenate them with the prompts completion_ids = [torch.tensor(ids, device=device) for ids in completion_ids] completion_ids = pad(completion_ids, padding_value=self.pad_token_id)