diff --git a/docs/guides/yarn-long-context.md b/docs/guides/yarn-long-context.md index cb45d968699..150b442a3fe 100644 --- a/docs/guides/yarn-long-context.md +++ b/docs/guides/yarn-long-context.md @@ -62,7 +62,7 @@ When `hf_config_overrides` are present, NeMo RL's Megatron setup: 2. Computes a stable hash over `hf_config_overrides` and appends it to the converted-checkpoint directory name (`__hfovr_`). Different override sets therefore produce separate cached checkpoints and will not collide. 3. Re-imports the HF model into Megatron format if no cached checkpoint at that path exists. -The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. +The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. They are merged into `policy.generation.vllm_kwargs.hf_overrides`: keys set only on the generation side are kept, and setting the same key to different values in both places raises an error at setup, so set YaRN keys only in `policy.hf_config_overrides`. ## Forcing a Fresh Conversion diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index dfe17422618..d6e1a7b90ad 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -68,6 +68,7 @@ from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration from nemo_rl.models.generation.vllm.config import ( VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -544,8 +545,9 @@ def setup( generation_config = cast(VllmConfig, generation_config) if "vllm_cfg" in generation_config: ## make vllm hf overrides match the training policy - generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get( - "hf_config_overrides", {} + merge_policy_hf_overrides( + generation_config["vllm_kwargs"], + policy_config.get("hf_config_overrides"), ) if enable_nemo_gym: deferred_vllm = VllmGeneration( diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index cb0094cb3c6..5927717c8d1 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -147,6 +147,7 @@ from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -1812,7 +1813,7 @@ def init_megatron_generation_task(policy_future): vllm_kwargs = generation_config.setdefault("vllm_kwargs", {}) ## make vllm hf overrides match the training policy - vllm_kwargs["hf_overrides"] = policy_config.get("hf_config_overrides", {}) + merge_policy_hf_overrides(vllm_kwargs, policy_config.get("hf_config_overrides")) if enable_nemo_gym: # ---- NeMo Gym: reserve vLLM ports up-front so we can hand the diff --git a/nemo_rl/algorithms/ppo.py b/nemo_rl/algorithms/ppo.py index 3685bf19400..5405fc88077 100644 --- a/nemo_rl/algorithms/ppo.py +++ b/nemo_rl/algorithms/ppo.py @@ -99,6 +99,7 @@ from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration from nemo_rl.models.generation.vllm.config import ( VLLM_SPARSE_REFIT_TRANSPORTS, + merge_policy_hf_overrides, normalize_nvfp4_pertoken_policy_config, normalize_vllm_refit_config, ) @@ -957,8 +958,8 @@ def initialize_generation_with_policy( ) ## make vllm hf overrides match the training policy - generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get( - "hf_config_overrides", {} + merge_policy_hf_overrides( + generation_config["vllm_kwargs"], policy_config.get("hf_config_overrides") ) policy_generation, policy, value_model = initialize_generation_with_policy( diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index ac8872ced81..21343c2d0d7 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -144,7 +144,10 @@ from nemo_rl.models.generation.sglang.config import SGLangConfig from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration from nemo_rl.models.generation.vllm import VllmGeneration -from nemo_rl.models.generation.vllm.config import VllmConfig +from nemo_rl.models.generation.vllm.config import ( + VllmConfig, + merge_policy_hf_overrides, +) from nemo_rl.models.megatron.router_replay import ( configure_vllm_for_router_replay, router_replay_enabled, @@ -639,8 +642,9 @@ def _build_generation( if backend == "vllm": vllm_config = cast(VllmConfig, generation_config) - vllm_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - master_config.policy.get("hf_config_overrides", {}) + merge_policy_hf_overrides( + vllm_config.setdefault("vllm_kwargs", {}), + master_config.policy.get("hf_config_overrides"), ) configure_vllm_for_router_replay(master_config.policy) gen = VllmGeneration( diff --git a/nemo_rl/models/generation/vllm/config.py b/nemo_rl/models/generation/vllm/config.py index 256119e9739..deaa02f5432 100644 --- a/nemo_rl/models/generation/vllm/config.py +++ b/nemo_rl/models/generation/vllm/config.py @@ -45,6 +45,38 @@ REFITTABLE_FP8_KV_CACHE_DTYPES = frozenset({"fp8", "fp8_e4m3"}) +def merge_policy_hf_overrides( + vllm_kwargs: dict[str, Any], policy_hf_config_overrides: Mapping[str, Any] | None +) -> None: + """Sync ``vllm_kwargs["hf_overrides"]`` with ``policy.hf_config_overrides`` in place. + + The trainer and vLLM must load the same HF config, so a key set to + different values in both dicts raises a ``ValueError`` naming it. Keys only set + under ``policy.generation.vllm_kwargs.hf_overrides`` survive: they exist to + correct the generation engine's view of the HF config without touching the + trainer, and a wholesale ``vllm_kwargs["hf_overrides"] = ...`` silently + discarded them (#3728). Mirrors ``_merge_fp8_kwargs`` one layer down. + """ + generation_overrides = vllm_kwargs.get("hf_overrides") or {} + if not isinstance(generation_overrides, Mapping): + raise ValueError("policy.generation.vllm_kwargs.hf_overrides must be a mapping") + policy_overrides = dict(policy_hf_config_overrides or {}) + + conflicts = sorted( + key + for key, value in policy_overrides.items() + if key in generation_overrides and generation_overrides[key] != value + ) + if conflicts: + raise ValueError( + f"policy.generation.vllm_kwargs.hf_overrides sets {conflicts} to values " + "that differ from policy.hf_config_overrides. The trainer and vLLM must " + "load the same HF config; set these keys only in policy.hf_config_overrides." + ) + + vllm_kwargs["hf_overrides"] = {**generation_overrides, **policy_overrides} + + # TODO(rohitrango): Move model-specific video fields behind ProcessorInterface. class VllmVideoConfig(BaseModel): """Video sampling contract shared by policy preprocessing and vLLM.""" diff --git a/tests/unit/models/generation/test_vllm_policy_hf_overrides.py b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py new file mode 100644 index 00000000000..620e6cf5a2d --- /dev/null +++ b/tests/unit/models/generation/test_vllm_policy_hf_overrides.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for syncing ``policy.hf_config_overrides`` into vLLM kwargs. + +The algorithm entry points used to assign ``vllm_kwargs["hf_overrides"]`` +wholesale, which silently dropped any generation-only overrides the user set +under ``policy.generation.vllm_kwargs.hf_overrides`` (#3728). +""" + +import warnings + +import pytest + +from nemo_rl.models.generation.vllm.config import merge_policy_hf_overrides + + +def test_generation_only_overrides_survive(): + vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}} + + merge_policy_hf_overrides(vllm_kwargs, None) + + assert vllm_kwargs["hf_overrides"] == {"max_position_embeddings": 8192} + + +def test_policy_overrides_are_added_without_conflict(): + vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}} + + with warnings.catch_warnings(): + warnings.simplefilter("error") + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 1_000_000.0}) + + assert vllm_kwargs["hf_overrides"] == { + "max_position_embeddings": 8192, + "rope_theta": 1_000_000.0, + } + + +def test_conflicting_values_raise(): + vllm_kwargs = {"hf_overrides": {"rope_theta": 1.0, "other": "keep"}} + + with pytest.raises(ValueError, match=r"\['rope_theta'\]"): + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0}) + + +def test_equal_values_do_not_warn(): + vllm_kwargs = {"hf_overrides": {"rope_theta": 2.0}} + + with warnings.catch_warnings(): + warnings.simplefilter("error") + merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0}) + + assert vllm_kwargs["hf_overrides"] == {"rope_theta": 2.0} + + +@pytest.mark.parametrize("existing", [{}, {"hf_overrides": None}, {"hf_overrides": {}}]) +def test_no_overrides_anywhere_yields_empty_mapping(existing): + vllm_kwargs = dict(existing) + + merge_policy_hf_overrides(vllm_kwargs, {}) + + assert vllm_kwargs["hf_overrides"] == {} + + +def test_policy_overrides_are_copied_not_aliased(): + policy_overrides = {"rope_theta": 2.0} + vllm_kwargs = {} + + merge_policy_hf_overrides(vllm_kwargs, policy_overrides) + vllm_kwargs["hf_overrides"]["extra"] = 1 + + assert policy_overrides == {"rope_theta": 2.0} + + +def test_non_mapping_generation_overrides_are_rejected(): + with pytest.raises(ValueError, match="must be a mapping"): + merge_policy_hf_overrides({"hf_overrides": "oops"}, {})