Skip to content

Commit 8dfeaf5

Browse files
committed
fix(vlm): materialize video frame contract in all runners
Signed-off-by: ehosseiniasl <ehsan.hosseiniasl@gmail.com>
1 parent 39381a4 commit 8dfeaf5

4 files changed

Lines changed: 68 additions & 0 deletions

File tree

‎examples/run_grpo_single_controller.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
configure_generation_config,
4747
maybe_configure_engine_reaping_env,
4848
)
49+
from nemo_rl.models.generation.vllm.config import materialize_vllm_video_config
4950
from nemo_rl.models.policy.draft_config import draft_refit_enabled
5051
from nemo_rl.telemetry.instrumentation import setup_span, startup_span
5152
from nemo_rl.telemetry.setup import init_telemetry_driver, shutdown_telemetry
@@ -96,6 +97,10 @@ def main() -> None:
9697

9798
config = OmegaConf.to_container(config, resolve=True)
9899
config = MasterConfig(**config)
100+
# Keep the processor, dataset and vLLM MediaIO on one frame-count contract.
101+
# Without this, vLLM's VideoMediaIO silently falls back to 32 frames even
102+
# when a video recipe requests (and caches) 64 frames.
103+
materialize_vllm_video_config(config.policy, config.data)
99104
print("Applied CLI overrides")
100105

101106
if is_ppo_run(config):

‎examples/run_vlm_grpo.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
)
3030
from nemo_rl.distributed.virtual_cluster import init_ray
3131
from nemo_rl.models.generation import configure_generation_config
32+
from nemo_rl.models.generation.vllm.config import materialize_vllm_video_config
3233
from nemo_rl.utils.config import (
3334
load_config,
3435
parse_hydra_overrides,
@@ -73,6 +74,10 @@ def main() -> None:
7374

7475
config = OmegaConf.to_container(config, resolve=True)
7576
config = MasterConfig(**config)
77+
# Materialize the video sampling contract before constructing the
78+
# processor or vLLM workers. In particular, this overrides vLLM
79+
# VideoMediaIO's independent 32-frame default for 64-frame recipes.
80+
materialize_vllm_video_config(config.policy, config.data)
7681
print("Applied CLI overrides")
7782

7883
# Print config

‎tests/unit/single_controller/test_entrypoint.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ def main_context(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
3535
"draft": Eagle3DraftConfig(enabled=False),
3636
"megatron_cfg": {"mtp_num_layers": 2},
3737
},
38+
data={},
3839
env={},
3940
data_plane={"enabled": True, "impl": "transfer_queue", "backend": "simple"},
4041
logger={"log_dir": "/tmp/logs"},
@@ -50,6 +51,7 @@ def main_context(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
5051
)
5152
configured_generation = {"backend": "vllm", "_mtp_weights_from_refit": True}
5253
configure_generation = MagicMock(return_value=configured_generation)
54+
materialize_video_config = MagicMock()
5355
actor = SimpleNamespace(run=SimpleNamespace(remote=MagicMock(return_value="run")))
5456
actor_args = SimpleNamespace(
5557
env_handles={},
@@ -92,6 +94,11 @@ def main_context(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
9294
"configure_generation_config",
9395
configure_generation,
9496
)
97+
monkeypatch.setattr(
98+
run_grpo_single_controller,
99+
"materialize_vllm_video_config",
100+
materialize_video_config,
101+
)
95102
monkeypatch.setattr(
96103
run_grpo_single_controller,
97104
"setup_single_controller",
@@ -116,6 +123,7 @@ def main_context(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
116123
configure_generation=configure_generation,
117124
configured_generation=configured_generation,
118125
generation_config=generation_config,
126+
materialize_video_config=materialize_video_config,
119127
ray_get=ray_get,
120128
ray_wait=ray_wait,
121129
setup_single_controller=setup_single_controller,
@@ -173,6 +181,16 @@ def test_main_configures_generation_for_trained_mtp(
173181
)
174182

175183

184+
def test_main_materializes_video_contract_before_worker_setup(
185+
main_context: SimpleNamespace,
186+
) -> None:
187+
run_grpo_single_controller.main()
188+
189+
main_context.materialize_video_config.assert_called_once_with(
190+
main_context.config.policy, main_context.config.data
191+
)
192+
193+
176194
def test_main_accepts_policy_without_draft_config(
177195
main_context: SimpleNamespace,
178196
) -> None:
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
3+
from argparse import Namespace
4+
from types import SimpleNamespace
5+
from unittest.mock import MagicMock
6+
7+
import pytest
8+
9+
from examples import run_vlm_grpo
10+
11+
12+
def test_vlm_entrypoint_materializes_video_contract(monkeypatch):
13+
"""The VLM runner must override vLLM VideoMediaIO's 32-frame default."""
14+
15+
class Materialized(RuntimeError):
16+
pass
17+
18+
config = SimpleNamespace(policy={"generation": {}}, data={})
19+
materialize = MagicMock(side_effect=Materialized)
20+
21+
monkeypatch.setattr(
22+
run_vlm_grpo,
23+
"parse_args",
24+
lambda: (Namespace(config="config.yaml"), []),
25+
)
26+
monkeypatch.setattr(run_vlm_grpo, "load_config", lambda _: {})
27+
monkeypatch.setattr(
28+
run_vlm_grpo.OmegaConf,
29+
"to_container",
30+
lambda *_args, **_kwargs: {},
31+
)
32+
monkeypatch.setattr(run_vlm_grpo, "MasterConfig", lambda **_: config)
33+
monkeypatch.setattr(
34+
run_vlm_grpo, "materialize_vllm_video_config", materialize
35+
)
36+
37+
with pytest.raises(Materialized):
38+
run_vlm_grpo.main()
39+
40+
materialize.assert_called_once_with(config.policy, config.data)

0 commit comments

Comments
 (0)