Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/guides/yarn-long-context.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ When `hf_config_overrides` are present, NeMo RL's Megatron setup:
2. Computes a stable hash over `hf_config_overrides` and appends it to the converted-checkpoint directory name (`<model>__hfovr_<hash>`). Different override sets therefore produce separate cached checkpoints and will not collide.
3. Re-imports the HF model into Megatron format if no cached checkpoint at that path exists.

The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer.
The same `hf_config_overrides` are also propagated to vLLM during generation, so the rollout engine applies the identical YaRN settings as the trainer. They are merged into `policy.generation.vllm_kwargs.hf_overrides`: keys set only on the generation side are kept, and setting the same key to different values in both places raises an error at setup, so set YaRN keys only in `policy.hf_config_overrides`.

## Forcing a Fresh Conversion

Expand Down
6 changes: 4 additions & 2 deletions nemo_rl/algorithms/distillation.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration
from nemo_rl.models.generation.vllm.config import (
VLLM_SPARSE_REFIT_TRANSPORTS,
merge_policy_hf_overrides,
normalize_nvfp4_pertoken_policy_config,
normalize_vllm_refit_config,
)
Expand Down Expand Up @@ -544,8 +545,9 @@ def setup(
generation_config = cast(VllmConfig, generation_config)
if "vllm_cfg" in generation_config:
## make vllm hf overrides match the training policy
generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get(
"hf_config_overrides", {}
merge_policy_hf_overrides(
generation_config["vllm_kwargs"],
policy_config.get("hf_config_overrides"),
)
if enable_nemo_gym:
deferred_vllm = VllmGeneration(
Expand Down
3 changes: 2 additions & 1 deletion nemo_rl/algorithms/grpo.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@
from nemo_rl.models.generation.vllm.config import (
REFITTABLE_FP8_KV_CACHE_DTYPES,
VLLM_SPARSE_REFIT_TRANSPORTS,
merge_policy_hf_overrides,
normalize_nvfp4_pertoken_policy_config,
normalize_vllm_refit_config,
)
Expand Down Expand Up @@ -1812,7 +1813,7 @@ def init_megatron_generation_task(policy_future):
vllm_kwargs = generation_config.setdefault("vllm_kwargs", {})

## make vllm hf overrides match the training policy
vllm_kwargs["hf_overrides"] = policy_config.get("hf_config_overrides", {})
merge_policy_hf_overrides(vllm_kwargs, policy_config.get("hf_config_overrides"))

if enable_nemo_gym:
# ---- NeMo Gym: reserve vLLM ports up-front so we can hand the
Expand Down
5 changes: 3 additions & 2 deletions nemo_rl/algorithms/ppo.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration
from nemo_rl.models.generation.vllm.config import (
VLLM_SPARSE_REFIT_TRANSPORTS,
merge_policy_hf_overrides,
normalize_nvfp4_pertoken_policy_config,
normalize_vllm_refit_config,
)
Expand Down Expand Up @@ -957,8 +958,8 @@ def initialize_generation_with_policy(
)

## make vllm hf overrides match the training policy
generation_config["vllm_kwargs"]["hf_overrides"] = policy_config.get(
"hf_config_overrides", {}
merge_policy_hf_overrides(
generation_config["vllm_kwargs"], policy_config.get("hf_config_overrides")
)

policy_generation, policy, value_model = initialize_generation_with_policy(
Expand Down
10 changes: 7 additions & 3 deletions nemo_rl/algorithms/single_controller_utils/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,10 @@
from nemo_rl.models.generation.sglang.config import SGLangConfig
from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration
from nemo_rl.models.generation.vllm import VllmGeneration
from nemo_rl.models.generation.vllm.config import VllmConfig
from nemo_rl.models.generation.vllm.config import (
VllmConfig,
merge_policy_hf_overrides,
)
from nemo_rl.models.megatron.router_replay import (
configure_vllm_for_router_replay,
router_replay_enabled,
Expand Down Expand Up @@ -639,8 +642,9 @@ def _build_generation(

if backend == "vllm":
vllm_config = cast(VllmConfig, generation_config)
vllm_config.setdefault("vllm_kwargs", {})["hf_overrides"] = (
master_config.policy.get("hf_config_overrides", {})
merge_policy_hf_overrides(
vllm_config.setdefault("vllm_kwargs", {}),
master_config.policy.get("hf_config_overrides"),
)
configure_vllm_for_router_replay(master_config.policy)
gen = VllmGeneration(
Expand Down
32 changes: 32 additions & 0 deletions nemo_rl/models/generation/vllm/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,38 @@
REFITTABLE_FP8_KV_CACHE_DTYPES = frozenset({"fp8", "fp8_e4m3"})


def merge_policy_hf_overrides(
vllm_kwargs: dict[str, Any], policy_hf_config_overrides: Mapping[str, Any] | None
) -> None:
"""Sync ``vllm_kwargs["hf_overrides"]`` with ``policy.hf_config_overrides`` in place.

The trainer and vLLM must load the same HF config, so a key set to
different values in both dicts raises a ``ValueError`` naming it. Keys only set
under ``policy.generation.vllm_kwargs.hf_overrides`` survive: they exist to
correct the generation engine's view of the HF config without touching the
trainer, and a wholesale ``vllm_kwargs["hf_overrides"] = ...`` silently
discarded them (#3728). Mirrors ``_merge_fp8_kwargs`` one layer down.
"""
generation_overrides = vllm_kwargs.get("hf_overrides") or {}
if not isinstance(generation_overrides, Mapping):
raise ValueError("policy.generation.vllm_kwargs.hf_overrides must be a mapping")
policy_overrides = dict(policy_hf_config_overrides or {})

conflicts = sorted(
key
for key, value in policy_overrides.items()
if key in generation_overrides and generation_overrides[key] != value
)
if conflicts:
raise ValueError(
f"policy.generation.vllm_kwargs.hf_overrides sets {conflicts} to values "
"that differ from policy.hf_config_overrides. The trainer and vLLM must "
"load the same HF config; set these keys only in policy.hf_config_overrides."
)
Comment on lines +70 to +75

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

C1 · 1 action item.
TL;DR — when both override dicts set one key to different values, vLLM gets the policy value and the run goes on with only a warning in the driver log. Elsewhere the repo raises a setup-time ValueError when two user settings disagree.

For example, policy.hf_config_overrides.rope_theta: 1e6 plus policy.generation.vllm_kwargs.hf_overrides.rope_theta: 5e5: the run starts and vLLM uses 1e6. The user asked for two different values and is never asked to pick one.

Other places in the repo reject the config instead of picking a winner:

No recipe on any public branch sets one key to different values in these two dicts, so a raise blocks no existing config.

AI-1

Could you replace the warning with a ValueError that names the keys and the fix?

    if conflicts:
        raise ValueError(
            f"policy.generation.vllm_kwargs.hf_overrides sets {conflicts} to values "
            "that differ from policy.hf_config_overrides. The trainer and vLLM must "
            "load the same HF config; set these keys only in policy.hf_config_overrides."
        )

This is a plain block, not a one-click suggestion, because three other spots describe the warning and need to change with it:

Where the new error fires in distillation and single-controller setup (no action needed)

With a raise, where the call sits starts to matter:

Moving these two calls earlier, before any model is built (distillation already has an early vLLM block), would make the error appear first. Not part of this ask.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in ff378e1: conflicting keys now raise your ValueError (names the keys and says to set them only in policy.hf_config_overrides). Updated the docstring, replaced the warn test with test_conflicting_values_raise (pytest.raises(ValueError, match=r"\['rope_theta'\]")), and changed the YaRN guide sentence. Equal values and generation-only keys behave as before. I also checked that no config under examples/configs or research/ sets vllm_kwargs.hf_overrides, so nothing existing trips it. Left the call-site ordering in distillation / non-colocated SC as is, per your note. Thanks for the careful review and for deferring #4949.


vllm_kwargs["hf_overrides"] = {**generation_overrides, **policy_overrides}


# TODO(rohitrango): Move model-specific video fields behind ProcessorInterface.
class VllmVideoConfig(BaseModel):
"""Video sampling contract shared by policy preprocessing and vLLM."""
Expand Down
88 changes: 88 additions & 0 deletions tests/unit/models/generation/test_vllm_policy_hf_overrides.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Regression tests for syncing ``policy.hf_config_overrides`` into vLLM kwargs.

The algorithm entry points used to assign ``vllm_kwargs["hf_overrides"]``
wholesale, which silently dropped any generation-only overrides the user set
under ``policy.generation.vllm_kwargs.hf_overrides`` (#3728).
"""

import warnings

import pytest

from nemo_rl.models.generation.vllm.config import merge_policy_hf_overrides


def test_generation_only_overrides_survive():
vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}}

merge_policy_hf_overrides(vllm_kwargs, None)

assert vllm_kwargs["hf_overrides"] == {"max_position_embeddings": 8192}


def test_policy_overrides_are_added_without_conflict():
vllm_kwargs = {"hf_overrides": {"max_position_embeddings": 8192}}

with warnings.catch_warnings():
warnings.simplefilter("error")
merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 1_000_000.0})

assert vllm_kwargs["hf_overrides"] == {
"max_position_embeddings": 8192,
"rope_theta": 1_000_000.0,
}


def test_conflicting_values_raise():
vllm_kwargs = {"hf_overrides": {"rope_theta": 1.0, "other": "keep"}}

with pytest.raises(ValueError, match=r"\['rope_theta'\]"):
merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0})


def test_equal_values_do_not_warn():
vllm_kwargs = {"hf_overrides": {"rope_theta": 2.0}}

with warnings.catch_warnings():
warnings.simplefilter("error")
merge_policy_hf_overrides(vllm_kwargs, {"rope_theta": 2.0})

assert vllm_kwargs["hf_overrides"] == {"rope_theta": 2.0}


@pytest.mark.parametrize("existing", [{}, {"hf_overrides": None}, {"hf_overrides": {}}])
def test_no_overrides_anywhere_yields_empty_mapping(existing):
vllm_kwargs = dict(existing)

merge_policy_hf_overrides(vllm_kwargs, {})

assert vllm_kwargs["hf_overrides"] == {}


def test_policy_overrides_are_copied_not_aliased():
policy_overrides = {"rope_theta": 2.0}
vllm_kwargs = {}

merge_policy_hf_overrides(vllm_kwargs, policy_overrides)
vllm_kwargs["hf_overrides"]["extra"] = 1

assert policy_overrides == {"rope_theta": 2.0}


def test_non_mapping_generation_overrides_are_rejected():
with pytest.raises(ValueError, match="must be a mapping"):
merge_policy_hf_overrides({"hf_overrides": "oops"}, {})
Loading