From 9f6be7bb049f91af933ac4d146d094d45131c35a Mon Sep 17 00:00:00 2001 From: Shubhankar Tripathy Date: Sat, 24 Oct 2026 22:56:17 -0700 Subject: [PATCH 1/2] fix(vllm): merge policy hf_config_overrides into generation hf_overrides Six call sites in grpo, ppo, distillation and the single-controller setup propagated `policy.hf_config_overrides` to the generation engine by wholesale assignment: vllm_kwargs["hf_overrides"] = policy_config.get("hf_config_overrides", {}) Anything the user set under `policy.generation.vllm_kwargs.hf_overrides` was discarded silently, even when the policy declared no overrides of its own. Generation-only overrides are needed when the rollout engine's view of the HF config must be corrected without changing the trainer (e.g. RoPE scaling for a trust_remote_code model); dropping them made generation and trainer logprobs disagree with nothing in the config or logs to show why. Add `merge_policy_hf_overrides` next to the other vLLM config normalizers and route all six sites through it. Policy overrides still win on conflicting keys (the trainer/generation RoPE invariant is unchanged), conflicts emit a warning naming the keys, and generation-only keys survive. This is the same overwrite-vs-merge bug `_merge_fp8_kwargs` already fixes one layer down (#1413 / #2904). Closes #3728 Co-Authored-By: Claude Fable 5.1 Signed-off-by: Shubhankar Tripathy --- docs/guides/yarn-long-context.md | 2 +- nemo_rl/algorithms/distillation.py | 6 +- nemo_rl/algorithms/grpo.py | 8 +- nemo_rl/algorithms/ppo.py | 5 +- .../single_controller_utils/setup.py | 15 ++-- nemo_rl/models/generation/vllm/config.py | 33 +++++++ tests/unit/algorithms/test_grpo.py | 11 +-- .../test_vllm_policy_hf_overrides.py | 90 +++++++++++++++++++ 8 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 tests/unit/models/generation/test_vllm_policy_hf_overrides.py diff --git a/docs/guides/yarn-long-context.md b/docs/guides/yarn-long-context.md index cb45d968699..58cb80b97a9 100644 --- a/docs/guides/yarn-long-context.md +++ b/docs/guides/yarn-long-context.md @@ -62,7 +62,7 @@ When `hf_config_overrides` are present, NeMo RL's Megatron setup: 2. Computes a stable hash over `hf_config_overrides` and appends it to the converted-checkpoint directory name (`__hfovr_`). Different override sets therefore produce separate cached checkpoints and will not collide. 3. Re-imports the HF model into Megatron format if no cached checkpoint at that path exists. -The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. +The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. They are merged into `policy.generation.vllm_kwargs.hf_overrides`: keys set only on the generation side are kept, and on a conflict the policy value wins with a warning. ## Forcing a Fresh Conversion diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index b861d73522c..003261ad0af 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -66,6 +66,7 @@ from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration from nemo_rl.models.generation.vllm.config import ( VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -531,8 +532,9 @@ def setup( generation_config = cast(VllmConfig, generation_config) if "vllm_cfg" in generation_config: ## make vllm hf overrides match the training policy - generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get( - "hf_config_overrides", {} + merge_policy_hf_overrides( + generation_config["vllm_kwargs"], + policy_config.get("hf_config_overrides"), ) if enable_nemo_gym: deferred_vllm = VllmGeneration( diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 5a96be314a3..e6c743a9b12 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -145,6 +145,7 @@ from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -716,8 +717,9 @@ def setup( "policy.generation.backend='dynamo'; managed Dynamo drains " "rollouts before layerwise weight refit" ) - generation_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} + merge_policy_hf_overrides( + generation_config.setdefault("vllm_kwargs", {}), + policy_config.get("hf_config_overrides"), ) generation_config = DynamoConfig.model_validate(generation_config).model_dump() policy_config["generation"] = generation_config @@ -1759,7 +1761,7 @@ def init_megatron_generation_task(policy_future): vllm_kwargs = generation_config.setdefault("vllm_kwargs", {}) ## make vllm hf overrides match the training policy - vllm_kwargs["hf_overrides"] = policy_config.get("hf_config_overrides", {}) + merge_policy_hf_overrides(vllm_kwargs, policy_config.get("hf_config_overrides")) if enable_nemo_gym: # ---- NeMo Gym: reserve vLLM ports up-front so we can hand the diff --git a/nemo_rl/algorithms/ppo.py b/nemo_rl/algorithms/ppo.py index f5b19f3f2d6..30957b56dcf 100644 --- a/nemo_rl/algorithms/ppo.py +++ b/nemo_rl/algorithms/ppo.py @@ -98,6 +98,7 @@ from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration from nemo_rl.models.generation.vllm.config import ( VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -950,8 +951,8 @@ def initialize_generation_with_policy( ) ## make vllm hf overrides match the training policy - generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get( - "hf_config_overrides", {} + merge_policy_hf_overrides( + generation_config["vllm_kwargs"], policy_config.get("hf_config_overrides") ) policy_generation, policy, value_model = initialize_generation_with_policy( diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index e3ffbb1ba11..f2e318e5d6e 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -135,7 +135,10 @@ from nemo_rl.models.generation.sglang.config import SGLangConfig from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration from nemo_rl.models.generation.vllm import VllmGeneration -from nemo_rl.models.generation.vllm.config import VllmConfig +from nemo_rl.models.generation.vllm.config import ( + VllmConfig, + merge_policy_hf_overrides, +) from nemo_rl.models.megatron.router_replay import ( configure_vllm_for_router_replay, router_replay_enabled, @@ -618,8 +621,9 @@ def _build_generation( if backend == "vllm": vllm_config = cast(VllmConfig, generation_config) - vllm_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - master_config.policy.get("hf_config_overrides", {}) + merge_policy_hf_overrides( + vllm_config.setdefault("vllm_kwargs", {}), + master_config.policy.get("hf_config_overrides"), ) configure_vllm_for_router_replay(master_config.policy) gen = VllmGeneration( @@ -1261,8 +1265,9 @@ def setup_single_controller( # Validate the complete managed-Dynamo boundary before allocating Ray # placement groups or starting any external services, as grpo.setup does. dynamo_config = cast(dict[str, Any], generation_config) - dynamo_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} + merge_policy_hf_overrides( + dynamo_config.setdefault("vllm_kwargs", {}), + policy_config.get("hf_config_overrides"), ) generation_config = cast( GenerationConfig, DynamoConfig.model_validate(dynamo_config).model_dump() diff --git a/nemo_rl/models/generation/vllm/config.py b/nemo_rl/models/generation/vllm/config.py index 9bb7ca05791..aa01809274d 100644 --- a/nemo_rl/models/generation/vllm/config.py +++ b/nemo_rl/models/generation/vllm/config.py @@ -42,6 +42,39 @@ REFITTABLE_FP8_KV_CACHE_DTYPES = frozenset({"fp8", "fp8_e4m3"}) +def merge_policy_hf_overrides( + vllm_kwargs: dict[str, Any], policy_hf_config_overrides: Mapping[str, Any] | None +) -> None: + """Sync ``vllm_kwargs["hf_overrides"]`` with ``policy.hf_config_overrides`` in place. + + The training policy's overrides are the source of truth, so on a key + collision the policy value wins and a warning names the key. Keys only set + under ``policy.generation.vllm_kwargs.hf_overrides`` survive: they exist to + correct the generation engine's view of the HF config without touching the + trainer, and a wholesale ``vllm_kwargs["hf_overrides"] = ...`` silently + discarded them (#3728). Mirrors ``_merge_fp8_kwargs`` one layer down. + """ + generation_overrides = vllm_kwargs.get("hf_overrides") or {} + if not isinstance(generation_overrides, Mapping): + raise ValueError("policy.generation.vllm_kwargs.hf_overrides must be a mapping") + policy_overrides = dict(policy_hf_config_overrides or {}) + + conflicts = sorted( + key + for key, value in policy_overrides.items() + if key in generation_overrides and generation_overrides[key] != value + ) + if conflicts: + warnings.warn( + "policy.hf_config_overrides overrides policy.generation.vllm_kwargs." + f"hf_overrides for {conflicts}; the training policy's value is used so " + "the generation engine stays in sync with the trainer.", + stacklevel=2, + ) + + vllm_kwargs["hf_overrides"] = {**generation_overrides, **policy_overrides} + + # TODO(rohitrango): Move model-specific video fields behind ProcessorInterface. class VllmVideoConfig(BaseModel): """Video sampling contract shared by policy preprocessing and vLLM.""" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 90fd42415ec..bc734a4250b 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -3367,7 +3367,7 @@ def test_dynamo_rejects_colocated_inference_before_setup_side_effects( "vllm_metrics_logger_interval": 1.0, "env_vars": {}, }, - "vllm_kwargs": {}, + "vllm_kwargs": {"hf_overrides": {"max_position_embeddings": 8192}}, "colocated": { "enabled": True, "resources": {"gpus_per_node": None, "num_nodes": None}, @@ -3404,10 +3404,11 @@ def test_dynamo_rejects_colocated_inference_before_setup_side_effects( ) mock_logger.assert_not_called() - assert ( - master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] - == (master_config.policy["hf_config_overrides"]) - ) + # Policy overrides are applied and generation-only keys survive (#3728). + assert master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] == { + "max_position_embeddings": 8192, + "rope_theta": 1_000_000.0, + } del master_config.policy["generation"]["vllm_kwargs"] del master_config.policy["hf_config_overrides"] diff --git a/tests/unit/models/generation/test_vllm_policy_hf_overrides.py b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py new file mode 100644 index 00000000000..848d68ef5a3 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py @@ -0,0 +1,90 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for syncing ``policy.hf_config_overrides`` into vLLM kwargs. + +The algorithm entry points used to assign ``vllm_kwargs["hf_overrides"]`` +wholesale, which silently dropped any generation-only overrides the user set +under ``policy.generation.vllm_kwargs.hf_overrides`` (#3728). +""" + +import warnings + +import pytest + +from nemo_rl.models.generation.vllm.config import merge_policy_hf_overrides + + +def test_generation_only_overrides_survive(): + vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}} + + merge_policy_hf_overrides(vllm_kwargs, None) + + assert vllm_kwargs["hf_overrides"] == {"max_position_embeddings": 8192} + + +def test_policy_overrides_are_added_without_conflict(): + vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}} + + with warnings.catch_warnings(): + warnings.simplefilter("error") + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 1_000_000.0}) + + assert vllm_kwargs["hf_overrides"] == { + "max_position_embeddings": 8192, + "rope_theta": 1_000_000.0, + } + + +def test_policy_wins_on_conflict_and_warns(): + vllm_kwargs = {"hf_overrides": {"rope_theta": 1.0, "other": "keep"}} + + with pytest.warns(UserWarning, match=r"\['rope_theta'\]"): + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0}) + + assert vllm_kwargs["hf_overrides"] == {"rope_theta": 2.0, "other": "keep"} + + +def test_equal_values_do_not_warn(): + vllm_kwargs = {"hf_overrides": {"rope_theta": 2.0}} + + with warnings.catch_warnings(): + warnings.simplefilter("error") + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0}) + + assert vllm_kwargs["hf_overrides"] == {"rope_theta": 2.0} + + +@pytest.mark.parametrize("existing", [{}, {"hf_overrides": None}, {"hf_overrides": {}}]) +def test_no_overrides_anywhere_yields_empty_mapping(existing): + vllm_kwargs = dict(existing) + + merge_policy_hf_overrides(vllm_kwargs, {}) + + assert vllm_kwargs["hf_overrides"] == {} + + +def test_policy_overrides_are_copied_not_aliased(): + policy_overrides = {"rope_theta": 2.0} + vllm_kwargs = {} + + merge_policy_hf_overrides(vllm_kwargs, policy_overrides) + vllm_kwargs["hf_overrides"]["extra"] = 1 + + assert policy_overrides == {"rope_theta": 2.0} + + +def test_non_mapping_generation_overrides_are_rejected(): + with pytest.raises(ValueError, match="must be a mapping"): + merge_policy_hf_overrides({"hf_overrides": "oops"}, {}) From ff378e1dcfe72826bf855de40d0fe9d43fbf9b2b Mon Sep 17 00:00:00 2001 From: Shubhankar Tripathy Date: Wed, 28 Oct 2026 00:59:32 -0700 Subject: [PATCH 2/2] fix(vllm): reject conflicting hf_overrides instead of warning Review on #4907 (C1): when policy.hf_config_overrides and policy.generation.vllm_kwargs.hf_overrides set one key to different values, the run started with only a driver-log warning and vLLM silently used the policy value. Elsewhere the repo rejects disagreeing user settings at setup (nvfp4_pertoken engine kwargs, Dynamo argv builder). merge_policy_hf_overrides now raises a ValueError naming the keys and the fix. Equal values and generation-only keys are unchanged. The docstring, the conflict test and the YaRN guide are updated to match. No shipped config sets vllm_kwargs.hf_overrides, so no existing recipe is affected. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Shubhankar Tripathy --- docs/guides/yarn-long-context.md | 2 +- nemo_rl/models/generation/vllm/config.py | 13 ++++++------- .../generation/test_vllm_policy_hf_overrides.py | 6 ++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/guides/yarn-long-context.md b/docs/guides/yarn-long-context.md index 58cb80b97a9..150b442a3fe 100644 --- a/docs/guides/yarn-long-context.md +++ b/docs/guides/yarn-long-context.md @@ -62,7 +62,7 @@ When `hf_config_overrides` are present, NeMo RL's Megatron setup: 2. Computes a stable hash over `hf_config_overrides` and appends it to the converted-checkpoint directory name (`__hfovr_`). Different override sets therefore produce separate cached checkpoints and will not collide. 3. Re-imports the HF model into Megatron format if no cached checkpoint at that path exists. -The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. They are merged into `policy.generation.vllm_kwargs.hf_overrides`: keys set only on the generation side are kept, and on a conflict the policy value wins with a warning. +The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. They are merged into `policy.generation.vllm_kwargs.hf_overrides`: keys set only on the generation side are kept, and setting the same key to different values in both places raises an error at setup, so set YaRN keys only in `policy.hf_config_overrides`. ## Forcing a Fresh Conversion diff --git a/nemo_rl/models/generation/vllm/config.py b/nemo_rl/models/generation/vllm/config.py index aa01809274d..f3c88ae6637 100644 --- a/nemo_rl/models/generation/vllm/config.py +++ b/nemo_rl/models/generation/vllm/config.py @@ -47,8 +47,8 @@ def merge_policy_hf_overrides( ) -> None: """Sync ``vllm_kwargs["hf_overrides"]`` with ``policy.hf_config_overrides`` in place. - The training policy's overrides are the source of truth, so on a key - collision the policy value wins and a warning names the key. Keys only set + The trainer and vLLM must load the same HF config, so a key set to + different values in both dicts raises a ``ValueError`` naming it. Keys only set under ``policy.generation.vllm_kwargs.hf_overrides`` survive: they exist to correct the generation engine's view of the HF config without touching the trainer, and a wholesale ``vllm_kwargs["hf_overrides"] = ...`` silently @@ -65,11 +65,10 @@ def merge_policy_hf_overrides( if key in generation_overrides and generation_overrides[key] != value ) if conflicts: - warnings.warn( - "policy.hf_config_overrides overrides policy.generation.vllm_kwargs." - f"hf_overrides for {conflicts}; the training policy's value is used so " - "the generation engine stays in sync with the trainer.", - stacklevel=2, + raise ValueError( + f"policy.generation.vllm_kwargs.hf_overrides sets {conflicts} to values " + "that differ from policy.hf_config_overrides. The trainer and vLLM must " + "load the same HF config; set these keys only in policy.hf_config_overrides." ) vllm_kwargs["hf_overrides"] = {**generation_overrides, **policy_overrides} diff --git a/tests/unit/models/generation/test_vllm_policy_hf_overrides.py b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py index 848d68ef5a3..620e6cf5a2d 100644 --- a/tests/unit/models/generation/test_vllm_policy_hf_overrides.py +++ b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py @@ -47,14 +47,12 @@ def test_policy_overrides_are_added_without_conflict(): } -def test_policy_wins_on_conflict_and_warns(): +def test_conflicting_values_raise(): vllm_kwargs = {"hf_overrides": {"rope_theta": 1.0, "other": "keep"}} - with pytest.warns(UserWarning, match=r"\['rope_theta'\]"): + with pytest.raises(ValueError, match=r"\['rope_theta'\]"): merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0}) - assert vllm_kwargs["hf_overrides"] == {"rope_theta": 2.0, "other": "keep"} - def test_equal_values_do_not_warn(): vllm_kwargs = {"hf_overrides": {"rope_theta": 2.0}}