Skip to content

Commit b28cff4

Browse files
committed
refactor(draft): add typed speculator contract
Signed-off-by: seonjinn <sna@nvidia.com>
1 parent f9a2b30 commit b28cff4

22 files changed

Lines changed: 342 additions & 57 deletions

‎examples/configs/grpo_math_1B.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,8 @@ policy:
314314
env_vars: null
315315

316316
draft:
317+
# Draft co-training speculator. Currently only eagle3 is supported.
318+
speculator_type: eagle3
317319
enabled: false
318320
model_name: null
319321
loss_weight: 0.1

‎examples/nemo_gym/run_grpo_nemo_gym.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ def main() -> None:
164164
"A generation config is required for GRPO"
165165
)
166166
has_refit_draft_weights = (
167-
"draft" in config.policy and config.policy["draft"]["enabled"]
167+
"draft" in config.policy and config.policy["draft"].enabled
168168
)
169169
trains_mtp = (
170170
"megatron_cfg" in config.policy

‎examples/run_grpo.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ def main() -> None:
111111
assert config.policy["generation"] is not None, (
112112
"A generation config is required for GRPO"
113113
)
114-
has_refit_draft_weights = bool(config.policy["draft"]["enabled"])
114+
has_refit_draft_weights = config.policy["draft"].enabled
115115
megatron_cfg = config.policy.get("megatron_cfg") or {}
116116
trains_mtp = bool(megatron_cfg.get("mtp_num_layers"))
117117
config.policy["generation"] = configure_generation_config(

‎examples/run_grpo_single_controller.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ def main() -> None:
121121
assert config.policy["generation"] is not None, (
122122
"A generation config is required for SC-driven async GRPO"
123123
)
124-
has_refit_draft_weights = bool(config.policy["draft"]["enabled"])
124+
has_refit_draft_weights = config.policy["draft"].enabled
125125
megatron_cfg = config.policy.get("megatron_cfg") or {}
126126
trains_mtp = bool(megatron_cfg.get("mtp_num_layers"))
127127
config.policy["generation"] = configure_generation_config(
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__ import annotations
16+
17+
from dataclasses import dataclass
18+
from typing import TYPE_CHECKING, Protocol, cast
19+
20+
from megatron.core.process_groups_config import ProcessGroupCollection
21+
from megatron.core.transformer import MegatronModule
22+
23+
from nemo_rl.models.megatron.draft.utils import build_draft_model
24+
from nemo_rl.models.policy.draft_config import Eagle3DraftConfig
25+
26+
if TYPE_CHECKING:
27+
from megatron.bridge.models.model_provider import ModelProviderMixin
28+
29+
30+
class DraftSpeculator(Protocol):
31+
"""Model-build operation selected by a configured draft speculator."""
32+
33+
config: Eagle3DraftConfig
34+
35+
def build_model(
36+
self,
37+
*,
38+
model_provider: ModelProviderMixin,
39+
pg_collection: ProcessGroupCollection,
40+
policy_model_chunk: MegatronModule,
41+
) -> MegatronModule | None:
42+
"""Build the draft model for the policy chunk that owns it."""
43+
44+
45+
@dataclass(frozen=True)
46+
class Eagle3Speculator:
47+
"""Current EAGLE-3 draft speculator."""
48+
49+
config: Eagle3DraftConfig
50+
51+
def build_model(
52+
self,
53+
*,
54+
model_provider: ModelProviderMixin,
55+
pg_collection: ProcessGroupCollection,
56+
policy_model_chunk: MegatronModule,
57+
) -> MegatronModule | None:
58+
"""Build the EAGLE-3 draft model with the existing implementation."""
59+
return build_draft_model(
60+
model_provider=model_provider,
61+
draft_config=self.config,
62+
pg_collection=pg_collection,
63+
policy_model_chunk=policy_model_chunk,
64+
)
65+
66+
67+
_SPECULATOR_FACTORIES: dict[str, type[Eagle3Speculator]] = {
68+
"eagle3": Eagle3Speculator,
69+
}
70+
71+
72+
def resolve_draft_speculator(
73+
config: Eagle3DraftConfig | None,
74+
) -> DraftSpeculator | None:
75+
"""Resolve an enabled draft configuration to its speculator."""
76+
if config is None or not config.enabled:
77+
return None
78+
return cast(
79+
DraftSpeculator,
80+
_SPECULATOR_FACTORIES[config.speculator_type](config),
81+
)

‎nemo_rl/models/megatron/draft/utils.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import re
1919
from dataclasses import dataclass
2020
from pathlib import Path
21-
from typing import Any, Callable, Mapping
21+
from typing import Callable, Mapping
2222

2323
import torch
2424
import torch.distributed as dist
@@ -28,6 +28,8 @@
2828
from megatron.core.utils import unwrap_model
2929
from torch import Tensor
3030

31+
from nemo_rl.models.policy.draft_config import Eagle3DraftConfig
32+
3133
StateDict = dict[str, Tensor]
3234
CheckpointLoader = Callable[[Path], StateDict]
3335

@@ -1227,12 +1229,12 @@ def register_draft_grad_norm_group() -> None:
12271229

12281230
def build_draft_model(
12291231
model_provider,
1230-
draft_config: dict[str, Any],
1232+
draft_config: Eagle3DraftConfig,
12311233
pg_collection: ProcessGroupCollection,
12321234
policy_model_chunk: MegatronModule,
12331235
) -> MegatronModule | None:
12341236
"""Build an Eagle draft model before parent mixed-precision/DDP wrapping."""
1235-
if not draft_config["enabled"]:
1237+
if not draft_config.enabled:
12361238
return None
12371239

12381240
from transformers import AutoConfig
@@ -1242,9 +1244,9 @@ def build_draft_model(
12421244
get_eagle3_aux_hidden_state_layers,
12431245
)
12441246

1245-
model_name = draft_config.get("model_name")
1247+
model_name = draft_config.model_name
12461248
hf_config = AutoConfig.from_pretrained(model_name).to_dict() if model_name else {}
1247-
draft_num_layers = draft_config.get("num_layers")
1249+
draft_num_layers = draft_config.num_layers
12481250
config = TransformerConfig(
12491251
normalization="RMSNorm",
12501252
activation_func=torch.nn.functional.silu,
@@ -1315,9 +1317,7 @@ def build_draft_model(
13151317
"eagle_aux_hidden_state_layer_ids", []
13161318
)
13171319
else:
1318-
config.eagle_aux_hidden_state_layer_ids = (
1319-
draft_config.get("aux_layer_indices") or []
1320-
)
1320+
config.eagle_aux_hidden_state_layer_ids = draft_config.aux_layer_indices or []
13211321
if (
13221322
config.use_aux_hidden_state
13231323
and len(config.eagle_aux_hidden_state_layer_ids) == 0

‎nemo_rl/models/megatron/setup.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -238,10 +238,10 @@ def _sync_distrib_opt(distrib_opt):
238238
RuntimeConfig,
239239
)
240240
from nemo_rl.models.megatron.draft.utils import (
241-
build_draft_model,
242241
find_draft_owner_chunk,
243242
get_attached_draft_model,
244243
)
244+
from nemo_rl.models.megatron.draft.training import resolve_draft_speculator
245245
from nemo_rl.models.megatron.memory_saver import inference_model_alloc_region
246246
from nemo_rl.models.megatron.router_replay import (
247247
clear_global_router_replay_instances,
@@ -1444,11 +1444,11 @@ def _create_draft_pre_wrap_hook(
14441444
preload_policy_from_pretrained: bool,
14451445
) -> Callable[[list[MegatronModule]], list[MegatronModule]]:
14461446
"""Create the hook that attaches draft weights before mixed-precision/DDP wrapping."""
1447-
draft_cfg = policy_cfg["draft"]
1447+
draft_speculator = resolve_draft_speculator(policy_cfg.get("draft"))
14481448

14491449
def draft_pre_wrap_hook(model: list[MegatronModule]) -> list[MegatronModule]:
14501450
"""Optionally preload the base policy, then attach the draft module to the owner chunk."""
1451-
if not draft_cfg["enabled"]:
1451+
if draft_speculator is None:
14521452
return model
14531453

14541454
# Base pretrained checkpoints do not contain draft weights, so load the
@@ -1483,9 +1483,8 @@ def draft_pre_wrap_hook(model: list[MegatronModule]) -> list[MegatronModule]:
14831483
)
14841484

14851485
pg_collection = get_pg_collection(model)
1486-
draft_model = build_draft_model(
1487-
megatron_cfg.model,
1488-
draft_config=draft_cfg,
1486+
draft_model = draft_speculator.build_model(
1487+
model_provider=megatron_cfg.model,
14891488
pg_collection=pg_collection,
14901489
policy_model_chunk=draft_owner,
14911490
)
@@ -1682,7 +1681,7 @@ def setup_model_and_optimizer(
16821681
pre_wrap_hook = []
16831682

16841683
use_peft = policy_cfg["megatron_cfg"].get("peft", {}).get("enabled", False)
1685-
draft_enabled = "draft" in policy_cfg and policy_cfg["draft"]["enabled"]
1684+
draft_enabled = "draft" in policy_cfg and policy_cfg["draft"].enabled
16861685
resume_checkpoint_exists = (
16871686
megatron_cfg.checkpoint.load is not None
16881687
and checkpoint_exists(megatron_cfg.checkpoint.load)

‎nemo_rl/models/megatron/train.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ def __call__(
547547
loss_fn=loss_fn_wrapped,
548548
prepare_fn=prepare_loss_input_wrapped,
549549
data_dict=data_dict,
550-
loss_weight=float(self.cfg["draft"]["loss_weight"]),
550+
loss_weight=float(self.cfg["draft"].loss_weight),
551551
vocab_parallel_rank=get_tensor_model_parallel_rank(),
552552
vocab_parallel_group=get_tensor_model_parallel_group(),
553553
context_parallel_group=get_context_parallel_group(),

‎nemo_rl/models/policy/__init__.py‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from typing import Any, Literal, NotRequired, TypedDict, Union
1616

1717
from nemo_rl.models.generation.interfaces import GenerationConfig
18+
from nemo_rl.models.policy.draft_config import Eagle3DraftConfig
1819
from nemo_rl.utils.checkpoint import PretrainedCheckpointConfig
1920

2021

@@ -481,22 +482,6 @@ class MegatronConfig(TypedDict):
481482
fp8_cfg: NotRequired[Fp8Config]
482483

483484

484-
class DraftConfigDisabled(TypedDict):
485-
"""Configuration shape for the disabled draft-model training path."""
486-
487-
enabled: Literal[False]
488-
489-
490-
class DraftConfig(TypedDict):
491-
"""Configuration for Eagle draft-model training alongside the policy model."""
492-
493-
enabled: Literal[True]
494-
model_name: NotRequired[str | None]
495-
loss_weight: NotRequired[float]
496-
num_layers: NotRequired[int | None]
497-
aux_layer_indices: NotRequired[list[int] | None]
498-
499-
500485
class TokenizerConfig(TypedDict):
501486
name: str
502487
chat_template: NotRequired[str]
@@ -570,7 +555,7 @@ class PolicyConfig(TypedDict):
570555
reward_model_cfg: NotRequired[RewardModelConfig]
571556
dtensor_cfg: DTensorConfig | DTensorConfigDisabled
572557
megatron_cfg: NotRequired[MegatronConfig | MegatronConfigDisabled]
573-
draft: NotRequired[DraftConfig | DraftConfigDisabled]
558+
draft: NotRequired[Eagle3DraftConfig]
574559
pretrained_checkpoint: NotRequired[PretrainedCheckpointConfig]
575560
router_replay: NotRequired[RouterReplayConfig | RouterReplayConfigDisabled]
576561
hf_config_overrides: NotRequired[dict[str, Any]]
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from typing import Literal
16+
17+
from pydantic import BaseModel
18+
19+
20+
class Eagle3DraftConfig(BaseModel, extra="allow"):
21+
"""Configuration for EAGLE-3 draft-model co-training with the policy."""
22+
23+
speculator_type: Literal["eagle3"] = "eagle3"
24+
enabled: bool = False
25+
model_name: str | None = None
26+
loss_weight: float = 0.1
27+
num_layers: int | None = None
28+
aux_layer_indices: list[int] | None = None

0 commit comments

Comments
 (0)