From c9e02bdae79452b7416b9aa82f5b3c5f256c133d Mon Sep 17 00:00:00 2001 From: edbeeching Date: Thu, 28 Aug 2025 12:38:53 +0000 Subject: [PATCH 01/12] add sleep mode for GRPO --- trl/trainer/grpo_config.py | 8 ++++++++ trl/trainer/grpo_trainer.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index 43c7e966aae..f24ec3fb1fa 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -140,6 +140,8 @@ class GRPOConfig(TrainingArguments): Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model implementation. + vllm_sleep_enabled (`bool`, *optional*, defaults to `False`): + Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation. > Parameters that control the training @@ -416,6 +418,12 @@ class GRPOConfig(TrainingArguments): "model implementation." }, ) + vllm_sleep_enabled: bool = field( + default=False, + metadata={ + "help": "Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation." + }, + ) vllm_guided_decoding_regex: Optional[str] = field( default=None, metadata={"help": "Regex for vLLM guided decoding. If `None` (default), guided decoding is disabled."}, diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 765ecb71508..4f60a1e4c36 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -835,7 +835,10 @@ def __init__( # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory max_num_batched_tokens=4096, model_impl=self.args.vllm_model_impl, + enable_sleep_mode=self.args.vllm_sleep_enabled, ) + if self.args.vllm_sleep_enabled: + self.llm.sleep(level=1) else: raise ValueError(f"vllm_mode must be either 'server' or 'colocate', got '{self.vllm_mode}'.") @@ -1426,6 +1429,11 @@ def _generate_and_score_completions( # Generate completions using either vLLM or regular generation if self.use_vllm: + if self.vllm_mode == "colocate" and self.args.vllm_sleep_enabled: + # wake up colocated vLLM instances if needed + torch.cuda.empty_cache() # required to avoid OOM in some cases + self.llm.wake_up() + # First, update the vLLM weights if needed if self.state.global_step != self._last_loaded_step: self._move_model_to_vllm() @@ -1533,7 +1541,10 @@ def _generate_and_score_completions( local_rank_in_group = torch.distributed.get_rank(group=self.tp_group) tp_slice = slice(local_rank_in_group * orig_size, (local_rank_in_group + 1) * orig_size) completion_ids = completion_ids[tp_slice] - + + if self.args.vllm_sleep_enabled: + self.llm.sleep(level=1) + # Pad the completions, and concatenate them with the prompts completion_ids = [torch.tensor(ids, device=device) for ids in completion_ids] completion_ids = pad(completion_ids, padding_value=self.pad_token_id) From 44f153329ab5ef088638d9a8e7485e3ee62c3d22 Mon Sep 17 00:00:00 2001 From: edbeeching Date: Thu, 28 Aug 2025 12:42:40 +0000 Subject: [PATCH 02/12] doc tip --- docs/source/grpo_trainer.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index e18149aabbc..a05b06413d1 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -251,6 +251,10 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. + +If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. + + From 654e97d0f5f2e81ea7f4431f21fefbc3d79ab87f Mon Sep 17 00:00:00 2001 From: edbeeching Date: Thu, 28 Aug 2025 12:43:20 +0000 Subject: [PATCH 03/12] precommit --- trl/trainer/grpo_trainer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 4f60a1e4c36..e788fb35aeb 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -1431,9 +1431,9 @@ def _generate_and_score_completions( if self.use_vllm: if self.vllm_mode == "colocate" and self.args.vllm_sleep_enabled: # wake up colocated vLLM instances if needed - torch.cuda.empty_cache() # required to avoid OOM in some cases + torch.cuda.empty_cache() # required to avoid OOM in some cases self.llm.wake_up() - + # First, update the vLLM weights if needed if self.state.global_step != self._last_loaded_step: self._move_model_to_vllm() @@ -1541,10 +1541,10 @@ def _generate_and_score_completions( local_rank_in_group = torch.distributed.get_rank(group=self.tp_group) tp_slice = slice(local_rank_in_group * orig_size, (local_rank_in_group + 1) * orig_size) completion_ids = completion_ids[tp_slice] - + if self.args.vllm_sleep_enabled: self.llm.sleep(level=1) - + # Pad the completions, and concatenate them with the prompts completion_ids = [torch.tensor(ids, device=device) for ids in completion_ids] completion_ids = pad(completion_ids, padding_value=self.pad_token_id) From 072f40c0239dba55354dc0a790c664435d69a24b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 29 Aug 2025 01:14:06 +0000 Subject: [PATCH 04/12] style --- trl/trainer/grpo_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index f24ec3fb1fa..83cdc78da36 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -141,7 +141,8 @@ class GRPOConfig(TrainingArguments): the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model implementation. vllm_sleep_enabled (`bool`, *optional*, defaults to `False`): - Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation. + Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken + for weight sync and generation. > Parameters that control the training @@ -421,7 +422,8 @@ class GRPOConfig(TrainingArguments): vllm_sleep_enabled: bool = field( default=False, metadata={ - "help": "Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation." + "help": "Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step " + "and woken for weight sync and generation." }, ) vllm_guided_decoding_regex: Optional[str] = field( From ab22c8dff9819c17e70c3ec7f6309cf86995be51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 29 Aug 2025 01:14:26 +0000 Subject: [PATCH 05/12] Add vLLM sleep mode documentation and reference in GRPO Trainer tips --- docs/source/grpo_trainer.md | 2 +- docs/source/reducing_memory_usage.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index a05b06413d1..e2fe393e2ec 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -252,7 +252,7 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. -If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. +If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing-memory-usage#vllm-sleep-mode). diff --git a/docs/source/reducing_memory_usage.md b/docs/source/reducing_memory_usage.md index 018738bdf34..1dd2363a5a3 100644 --- a/docs/source/reducing_memory_usage.md +++ b/docs/source/reducing_memory_usage.md @@ -267,3 +267,28 @@ training_args = RLOOConfig(..., ds3_gather_for_generation=False) This adjustment prevents model weights from being gathered, avoiding OOM errors, but it may result in slower generation speeds. + +## vLLM sleep mode + +When using vLLM as the generation backend, you can enable _sleep mode_ to offload vLLM parameters and cache to CPU RAM during the optimization step and reload them back to GPU VRAM when needed for weight synchronization and generation. + + + + +```python +from trl import GRPOConfig + +training_args = GRPOConfig(..., vllm_sleep_enabled=True) +``` + + + + +```python +from trl import RLOOConfig + +training_args = RLOOConfig(..., vllm_sleep_enabled=True) +``` + + + From 676303156d74d621a71589764a22d23283fb2a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Thu, 28 Aug 2025 18:15:16 -0700 Subject: [PATCH 06/12] Update docs/source/grpo_trainer.md Co-authored-by: Sergio Paniego Blanco --- docs/source/grpo_trainer.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index e2fe393e2ec..c6b10a582d1 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -252,6 +252,7 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. + If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing-memory-usage#vllm-sleep-mode). From e55b88a1c1ab04e978208034f3377f032f5f97a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Thu, 28 Aug 2025 18:15:40 -0700 Subject: [PATCH 07/12] Update docs/source/grpo_trainer.md Co-authored-by: Sergio Paniego Blanco --- docs/source/grpo_trainer.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index c6b10a582d1..8c842965bb7 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -254,6 +254,7 @@ If the recommended value does not work in your environment, we suggest adding a If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing-memory-usage#vllm-sleep-mode). + From adb473602111c5cc2dd6f16f19e84c4879d56bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 29 Aug 2025 01:18:54 +0000 Subject: [PATCH 08/12] remove nested tip --- docs/source/grpo_trainer.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index 8c842965bb7..30878ba1955 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -251,14 +251,10 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. - - If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing-memory-usage#vllm-sleep-mode). - - By default, GRPO uses `MASTER_ADDR=localhost` and `MASTER_PORT=12345` for vLLM, but you can override these values by setting the environment variables accordingly. From 415d187a819bbeccea927fef647d5028461152a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 29 Aug 2025 01:24:06 +0000 Subject: [PATCH 09/12] Fix link to vLLM sleep mode documentation in GRPO Trainer tips --- docs/source/grpo_trainer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/grpo_trainer.md b/docs/source/grpo_trainer.md index 30878ba1955..9ca20a6ef37 100644 --- a/docs/source/grpo_trainer.md +++ b/docs/source/grpo_trainer.md @@ -251,7 +251,7 @@ We provide a [HF Space](https://huggingface.co/spaces/trl-lib/recommend-vllm-mem If the recommended value does not work in your environment, we suggest adding a small buffer (e.g., +0.05 or +0.1) to the recommended value to ensure stability. -If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing-memory-usage#vllm-sleep-mode). +If you still find you are getting out-of-memory errors set `vllm_sleep_enabled` to True and the vllm parameters and cache will be offloaded during the optimization step. For more information, see [Reducing Memory Usage with vLLM Sleep Mode](reducing_memory_usage#vllm-sleep-mode). From 78f9d60bde5e1fbdd35aa452b12d1564b915ecd6 Mon Sep 17 00:00:00 2001 From: edbeeching Date: Fri, 29 Aug 2025 07:27:55 +0000 Subject: [PATCH 10/12] nits --- trl/trainer/grpo_config.py | 4 ++-- trl/trainer/grpo_trainer.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/trl/trainer/grpo_config.py b/trl/trainer/grpo_config.py index f24ec3fb1fa..3b6ff4ad4cf 100644 --- a/trl/trainer/grpo_config.py +++ b/trl/trainer/grpo_config.py @@ -140,7 +140,7 @@ class GRPOConfig(TrainingArguments): Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model implementation. - vllm_sleep_enabled (`bool`, *optional*, defaults to `False`): + vllm_enable_sleep_mode (`bool`, *optional*, defaults to `False`): Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation. > Parameters that control the training @@ -418,7 +418,7 @@ class GRPOConfig(TrainingArguments): "model implementation." }, ) - vllm_sleep_enabled: bool = field( + vllm_enable_sleep_mode: bool = field( default=False, metadata={ "help": "Whether to enable sleep mode for vLLM. If `True`, vLLM will sleep during the optimization step and woken for weight sync and generation." diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index e788fb35aeb..41765b6a98d 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -835,10 +835,10 @@ def __init__( # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory max_num_batched_tokens=4096, model_impl=self.args.vllm_model_impl, - enable_sleep_mode=self.args.vllm_sleep_enabled, + enable_sleep_mode=self.args.vllm_enable_sleep_mode, ) - if self.args.vllm_sleep_enabled: - self.llm.sleep(level=1) + if self.args.vllm_enable_sleep_mode: + self.llm.sleep(level=2) else: raise ValueError(f"vllm_mode must be either 'server' or 'colocate', got '{self.vllm_mode}'.") @@ -1429,7 +1429,7 @@ def _generate_and_score_completions( # Generate completions using either vLLM or regular generation if self.use_vllm: - if self.vllm_mode == "colocate" and self.args.vllm_sleep_enabled: + if self.vllm_mode == "colocate" and self.args.vllm_enable_sleep_mode: # wake up colocated vLLM instances if needed torch.cuda.empty_cache() # required to avoid OOM in some cases self.llm.wake_up() @@ -1542,8 +1542,8 @@ def _generate_and_score_completions( tp_slice = slice(local_rank_in_group * orig_size, (local_rank_in_group + 1) * orig_size) completion_ids = completion_ids[tp_slice] - if self.args.vllm_sleep_enabled: - self.llm.sleep(level=1) + if self.args.vllm_enable_sleep_mode: + self.llm.sleep(level=2) # Pad the completions, and concatenate them with the prompts completion_ids = [torch.tensor(ids, device=device) for ids in completion_ids] From 8c67f8e04266bdba02e47de12d42a6d99f1367b1 Mon Sep 17 00:00:00 2001 From: edbeeching Date: Fri, 29 Aug 2025 07:59:10 +0000 Subject: [PATCH 11/12] revert change to level --- trl/trainer/grpo_trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index 41765b6a98d..a6d797e791f 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -838,7 +838,7 @@ def __init__( enable_sleep_mode=self.args.vllm_enable_sleep_mode, ) if self.args.vllm_enable_sleep_mode: - self.llm.sleep(level=2) + self.llm.sleep(level=1) else: raise ValueError(f"vllm_mode must be either 'server' or 'colocate', got '{self.vllm_mode}'.") @@ -1543,7 +1543,7 @@ def _generate_and_score_completions( completion_ids = completion_ids[tp_slice] if self.args.vllm_enable_sleep_mode: - self.llm.sleep(level=2) + self.llm.sleep(level=1) # Pad the completions, and concatenate them with the prompts completion_ids = [torch.tensor(ids, device=device) for ids in completion_ids] From 01302812ea8ec4e893635a52dc4740ab3b23c020 Mon Sep 17 00:00:00 2001 From: edbeeching Date: Mon, 1 Sep 2025 07:38:04 +0000 Subject: [PATCH 12/12] dummy commit --- trl/trainer/grpo_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index a6d797e791f..eb2c3d8ba1f 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -832,7 +832,7 @@ def __init__( distributed_executor_backend="external_launcher", # Feed identical seed for tp groups to ensure sampling results are the same across workers seed=self.accelerator.process_index // self.vllm_tensor_parallel_size, - # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory + # Latest vLLM v1 memory profiler is misled by the high default value (i.e., 32768) - thinking there's not enough memory max_num_batched_tokens=4096, model_impl=self.args.vllm_model_impl, enable_sleep_mode=self.args.vllm_enable_sleep_mode,