From 036c279afb9fd7aafabc4e2bb00b287206ed7223 Mon Sep 17 00:00:00 2001 From: Jonas Yang Date: Mon, 2 Nov 2026 15:03:18 +0800 Subject: [PATCH 1/8] fix(single_controller): reject below-threshold finalizer groups before commit Dropping the slot after the row clears (previous commit) still left a window: while the clear RPCs are awaited the slot is ready, so a starved train pump can claim it and train on rows being cleared, and the later drop then fails with "unknown group_ids". Decide token_capture.min_valid_fraction_per_group inside the finalize cut, before commit_finalized. A rejected group is cleaned up like a structural finalizer drop (rows cleared, unready slot aborted), so it is never selectable and never enriched; drop_committed_group is no longer needed. Its replacement is taken in the same cut, and with rollout recovery the spare's ledger reservation is too (or the step is credited short), so no rollout snapshot sees the group gone without its replacement. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonas Yang --- nemo_rl/algorithms/single_controller.py | 186 ++++++++------ .../test_finalizer_lifecycle.py | 234 +++++++++++------- .../single_controller/test_rollout_pump.py | 61 +++-- 3 files changed, 299 insertions(+), 182 deletions(-) diff --git a/nemo_rl/algorithms/single_controller.py b/nemo_rl/algorithms/single_controller.py index 0163f77d4e6..6b48ab9ceca 100644 --- a/nemo_rl/algorithms/single_controller.py +++ b/nemo_rl/algorithms/single_controller.py @@ -52,7 +52,7 @@ import warnings from collections import deque from collections.abc import AsyncIterator, Iterator -from dataclasses import replace +from dataclasses import dataclass, replace from functools import partial from pathlib import Path from typing import ( @@ -250,6 +250,24 @@ _SUPERVISOR_DRAIN_TIMEOUT_S = 30.0 +@dataclass(frozen=True) +class _BelowValidFraction: + """A finalized group rejected by ``min_valid_fraction_per_group`` before commit. + + The group never became selectable. ``replacement`` is the spare prompt taken + in the same mutation cut that cleaned the group up (None when the step was + credited short instead), and ``lineage_group_id`` is its ledger reservation + when rollout recovery is enabled. ``reward_observations`` are the valid rows' + rewards, which the pump still tallies into the step's reward metrics. + """ + + valid_count: int + total_count: int + replacement: Optional[DatumSpec] + lineage_group_id: Optional[str] + reward_observations: list[FinalizedReward] + + def _latest_generation_values( metrics: dict[str, Any], metric_name: str ) -> dict[int, float]: @@ -1330,43 +1348,49 @@ async def _cleanup_known_finalization_request( ) as cut: await self._cleanup_known_finalization_request_unlocked(cut, request) - async def _discard_committed_group(self, request: "ReassemblyRequest") -> bool: - """Release a committed group the controller declines to train on. - - Returns: - True when the group was removed and its rows cleared. False when the train - pump claimed it for training before this ran, in which case the step is not - short and nothing was cleared. - """ - # Remove the ready slot before clearing rows; a missing slot means training owns it. - try: - await self._buffer.remove_group(request.group_id) - except ValueError: - print( - f" finalize: group {request.group_id} was claimed for training " - "before it could be discarded; its rows belong to that step now", - flush=True, - ) - return False - await self._cleanup_known_finalization_request(request) - # The group left the buffer unconsumed, so the post-step release for - # consumed groups never reaches its pending metrics and observations. - self._rollout_runtime.finalizer_metrics_by_group.pop(request.group_id, None) - self._rollout_runtime.finalizer_rewards_by_group.pop(request.group_id, None) - return True + def _valid_fraction_counts( + self, finalized: "FinalizedGroup" + ) -> Optional[tuple[int, int]]: + """Return (valid, total) when the group falls below the valid-fraction bar.""" + min_valid_fraction = ( + self._master_config.token_capture.min_valid_fraction_per_group + ) + if min_valid_fraction is None: + return None + if self._master_config.token_capture.forest_training: + # Compaction expands owners into unequal row counts; use the + # finalizer's owner totals. + valid_count = int(finalized.metrics["finalize/valid_logical_owner_count"]) + total_count = int(finalized.metrics["finalize/logical_owner_count"]) + else: + valid_count = finalized.valid_row_count + total_count = finalized.total_row_count + if total_count > 0 and valid_count / total_count < min_valid_fraction: + return valid_count, total_count + return None async def _finalize_with_actor( - self, request: "ReassemblyRequest", rollout_metrics: dict[str, Any] - ) -> Optional["FinalizedGroup"]: + self, + request: "ReassemblyRequest", + rollout_metrics: dict[str, Any], + *, + target_step: Optional[int], + replacements_used: int, + ) -> Union["FinalizedGroup", _BelowValidFraction, None]: """Finalize and index one group atomically with respect to TQ saves. Returns the committed FinalizedGroup once the group is committed to - the replay buffer (callers use logical-owner counts for forests, or row - counts for ordinary capture, to decide whether to keep it), or None when - the finalizer itself dropped it as a structural outcome (ownership - already cleaned up; the caller credits the step short). A low - valid-rollout fraction is no longer a finalizer-side drop -- the caller - decides that, since only the caller can source a replacement. + the replay buffer, or None when the finalizer itself dropped it as a + structural outcome (ownership already cleaned up; the caller credits + the step short). + + A group below ``token_capture.min_valid_fraction_per_group`` is + rejected before ``commit_finalized`` marks its slot ready, so the + sampler can never select it. Its cleanup, the spare prompt that + replaces it (or the step's shortfall credit), and that spare's ledger + reservation share the commit's mutation cut, so no snapshot sees the + group gone without its replacement. Returns a ``_BelowValidFraction`` + in that case. """ self._finalizer_waiters += 1 queue_depth = max( @@ -1387,6 +1411,7 @@ async def _finalize_with_actor( finalize_start = time.perf_counter() rpc_submitted = False actor_reusable = False + rejected: Optional[_BelowValidFraction] = None try: # The actor publishes canonical rows before returning metadata. Keep # the remote write, local replay-index update, and lineage hand-off in @@ -1446,6 +1471,33 @@ async def _finalize_with_actor( "finalizer returned no metadata for non-dropped group " f"{request.group_id}" ) + elif (counts := self._valid_fraction_counts(finalized)) is not None: + try: + await self._cleanup_known_finalization_request_unlocked( + cut, request + ) + except BaseException as cleanup_error: + raise RuntimeError( + "finalizer group fell below " + "min_valid_fraction_per_group and known-key cleanup " + f"failed for group {request.group_id}" + ) from cleanup_error + replacement = self._take_replacement(replacements_used) + lineage_group_id = None + if replacement is None: + self._credit_shortfall(target_step) + elif self._rollout_recovery_enabled: + lineage_group_id = self._rollout_manager.reserve_prompt_group( + cut, replacement, target_step=target_step + ) + rejected = _BelowValidFraction( + valid_count=counts[0], + total_count=counts[1], + replacement=replacement, + lineage_group_id=lineage_group_id, + reward_observations=list(finalized.reward_observations), + ) + committed = False else: extra_env_info: dict[str, Any] = {} if request.attribution_key is not None: @@ -1532,7 +1584,7 @@ async def _finalize_with_actor( self._available_finalizers.put_nowait(actor) finalize_total_ms = (time.perf_counter() - finalize_start) * 1000.0 if not committed: - return None + return rejected finalized.metrics.update( { "finalize/queue_wait_ms": queue_wait_ms, @@ -1753,7 +1805,10 @@ async def _dispatch_one_prompt( return request, rollout_metrics = captured finalized = await self._finalize_with_actor( - request, rollout_metrics + request, + rollout_metrics, + target_step=target_step, + replacements_used=replacements, ) if finalized is None: # Finalizer dropped the group as a structural @@ -1775,65 +1830,32 @@ async def _dispatch_one_prompt( replacement_index=replacements, zero_variance=rewards_carry_no_signal(capture_rewards), ) - min_valid_fraction = self._master_config.token_capture.min_valid_fraction_per_group - valid_count = total_count = 0 - below_threshold = False - if min_valid_fraction is not None: - if self._master_config.token_capture.forest_training: - # Compaction expands owners into unequal row - # counts; use the finalizer's owner totals. - valid_count = int( - finalized.metrics[ - "finalize/valid_logical_owner_count" - ] - ) - total_count = int( - finalized.metrics[ - "finalize/logical_owner_count" - ] - ) - else: - valid_count = finalized.valid_row_count - total_count = finalized.total_row_count - below_threshold = ( - total_count > 0 - and valid_count / total_count < min_valid_fraction - ) - if not below_threshold: - self._rollout_manager.stats.committed += 1 - ownership_transferred = True - break - # Only the controller can replace a published group below the training threshold. - try: - discarded = await self._discard_committed_group(request) - except BaseException as cleanup_error: - raise RuntimeError( - "finalizer group fell below " - "min_valid_fraction_per_group and " - "known-key cleanup failed for group " - f"{request.group_id}" - ) from cleanup_error - if not discarded: - # Training already owns the group and its permit; do not replace it. + if not isinstance(finalized, _BelowValidFraction): self._rollout_manager.stats.committed += 1 ownership_transferred = True break + # Too few rollouts verified to be worth training on. + # The finalize cut already cleaned the group up and + # took its replacement (or credited the step short). print( f" finalize: group {request.group_id} below " "min_valid_fraction_per_group " - f"({valid_count}/{total_count} < " - f"{min_valid_fraction}); seeking a replacement", + f"({finalized.valid_count}/{finalized.total_count} < " + f"{self._master_config.token_capture.min_valid_fraction_per_group}); " + + ( + "substituting a spare prompt" + if finalized.replacement is not None + else "no replacement; dropping the group" + ), flush=True, ) - replacement = self._take_replacement(replacements) - if replacement is None: + if finalized.replacement is None: self._rollout_manager.record_finalizer_dropped_prompt() self._buffer_capacity.release() - self._credit_shortfall(target_step) return replacements += 1 - prompt = replacement - lineage_group_id = None + prompt = finalized.replacement + lineage_group_id = finalized.lineage_group_id # A substitution is a fresh rollout, not a # continuation of the one that fell short, so it # observes the same pause a first dispatch does diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index 396d8765d73..a01def6dd67 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -18,6 +18,7 @@ import asyncio import threading +from collections import deque from types import SimpleNamespace from typing import Any from unittest.mock import ANY, AsyncMock, MagicMock @@ -33,6 +34,7 @@ from nemo_rl.algorithms.single_controller import ( RolloutRuntimeState, SingleControllerActor, + _BelowValidFraction, ) from nemo_rl.data_plane import KVBatchMeta from nemo_rl.data_plane.schema import ROLLOUT_METRICS, ROUTE_PLAN_TAG @@ -119,12 +121,15 @@ def _controller(actor: object) -> Any: ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._buffer = MagicMock() ctrl._buffer.commit_finalized = AsyncMock() - ctrl._buffer.remove_group = AsyncMock() ctrl._dp_client = _DataPlaneClient() ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._partition_id = "canonical" ctrl._master_config = SimpleNamespace( - token_capture=SimpleNamespace(staging_partition="staging"), + token_capture=SimpleNamespace( + staging_partition="staging", + min_valid_fraction_per_group=None, + forest_training=False, + ), grpo=SimpleNamespace(num_prompts_per_step=1), ) ctrl._trainer_version = 3 @@ -170,7 +175,9 @@ def test_successful_actor_finalization_returns_actor_and_transfers_ownership( ctrl = _controller(actor) request = _request(resolved_agent_name, attribution_key=attribution_key) - committed = asyncio.run(ctrl._finalize_with_actor(request, {})) + committed = asyncio.run( + ctrl._finalize_with_actor(request, {}, target_step=None, replacements_used=0) + ) assert committed is result assert finalize.calls == [request] @@ -213,7 +220,11 @@ def test_capture_rollout_metrics_ride_on_the_committed_meta() -> None: ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) metrics = {"agent/rollout_time_taken/mean": 12.5} - asyncio.run(ctrl._finalize_with_actor(_request(), metrics)) + asyncio.run( + ctrl._finalize_with_actor( + _request(), metrics, target_step=None, replacements_used=0 + ) + ) committed_meta = ctrl._buffer.commit_finalized.await_args.args[2] assert committed_meta.extra_info == {"existing": 1, ROLLOUT_METRICS: [metrics]} @@ -248,7 +259,11 @@ def test_finalizer_token_metrics_and_rollout_metrics_share_one_sidecar( result = _token_metrics_result(token_metrics) ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) - asyncio.run(ctrl._finalize_with_actor(_request(), rollout_metrics)) + asyncio.run( + ctrl._finalize_with_actor( + _request(), rollout_metrics, target_step=None, replacements_used=0 + ) + ) committed_meta = ctrl._buffer.commit_finalized.await_args.args[2] assert committed_meta.extra_info[ROLLOUT_METRICS] == [ @@ -262,7 +277,12 @@ def test_overlapping_finalizer_and_rollout_metrics_fail_the_commit() -> None: with pytest.raises(RuntimeError, match="both report"): asyncio.run( - ctrl._finalize_with_actor(_request(), {"gen_tokens_per_sample/mean": 1.0}) + ctrl._finalize_with_actor( + _request(), + {"gen_tokens_per_sample/mean": 1.0}, + target_step=None, + replacements_used=0, + ) ) ctrl._buffer.commit_finalized.assert_not_awaited() @@ -336,6 +356,8 @@ def score(received_meta: KVBatchMeta) -> None: ctrl._finalize_with_actor( _request(None if outcome == "missing_route" else "image_agent"), {}, + target_step=None, + replacements_used=0, ) ) try: @@ -401,105 +423,139 @@ def score(received_meta: KVBatchMeta) -> None: asyncio.run(exercise()) -def test_below_threshold_group_releases_pending_finalizer_state() -> None: - """A committed group dropped for min_valid_fraction_per_group leaves no state.""" - meta = KVBatchMeta( - partition_id="canonical", - task_name="train", - sample_ids=["group_g0"], - fields=["input_ids"], - sequence_lengths=[3], - tags=[{"weight_version": 3}], - ) - result = FinalizedGroup( - meta=meta, +def _below_threshold_result() -> FinalizedGroup: + # One of two rollouts verified: 0.5 < the 0.75 bar set by _below_threshold. + return FinalizedGroup( + meta=KVBatchMeta( + partition_id="canonical", + task_name="train", + sample_ids=["group_g0", "group_g1"], + fields=["input_ids"], + sequence_lengths=[3, 3], + tags=[{"weight_version": 3}, {"weight_version": 3}], + ), group_min_wv=3, group_max_wv=3, staging_keys=["group_g0/call"], - metrics={"finalize/invalid_row_rate": 0.0}, + metrics={"finalize/invalid_row_rate": 0.5}, reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + valid_row_count=1, + total_row_count=2, ) - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) - request = _request() - async def _below_threshold_branch() -> None: - # The same calls the rollout pump makes when - # valid_row_count / total_row_count < min_valid_fraction_per_group. - assert await ctrl._finalize_with_actor(request, {}) is result - assert "group" in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" in ctrl._rollout_runtime.finalizer_rewards_by_group - assert await ctrl._discard_committed_group(request) is True - asyncio.run(_below_threshold_branch()) +def _below_threshold(ctrl: Any, *, spares: list[dict[str, Any]]) -> None: + ctrl._master_config.token_capture.min_valid_fraction_per_group = 0.75 + ctrl._async_cfg = SimpleNamespace( + rollout_failure=SimpleNamespace( + on_dropped_prompt="replace", max_replacement_attempts=1 + ) + ) + ctrl._rollout_runtime.replacement_reserve = deque(spares) + ctrl._rollout_runtime.batch_shortfall = {} + ctrl._rollout_runtime.sampler_stamps_target_steps = True + ctrl._rollout_recovery_enabled = True + - ctrl._buffer.remove_group.assert_awaited_once_with("group") - ctrl._buffer.abort.assert_called_once_with("group") - assert "group" not in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" not in ctrl._rollout_runtime.finalizer_rewards_by_group +class _ReadyWatchingClient(_DataPlaneClient): + """Records whether any replay slot was ready while rows were being cleared.""" + + def __init__(self) -> None: + super().__init__() + self.buffer: TQReplayBuffer | None = None + self.ready_during_clear: list[bool] = [] + + async def clear_samples(self, *, sample_ids: list[str], partition_id: str) -> None: + assert self.buffer is not None + self.ready_during_clear.append(any(self.buffer.ready_list)) + await asyncio.sleep(0) # let any other task run mid-cleanup + await super().clear_samples(sample_ids=sample_ids, partition_id=partition_id) -def test_discarded_committed_group_leaves_no_ready_slot_in_the_buffer() -> None: - """abort() skips ready slots, so discarding must remove the committed slot itself.""" - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize())) - buffer = TQReplayBuffer( - ctrl._dp_client, +def test_below_threshold_group_never_becomes_selectable() -> None: + """A rejected group leaves the buffer without ever being marked ready.""" + enricher_calls: list[str] = [] + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[]) + client = _ReadyWatchingClient() + ctrl._dp_client = client + ctrl._buffer = TQReplayBuffer( + client, partition_id="canonical", pad_value_dict={"token_ids": 0}, include_message_violation_fields=False, require_routed_experts=False, ) - buffer.set_data_plane_checkpoint_barrier(ctrl._data_plane_checkpoint_barrier) - ctrl._buffer = buffer - request = _request() - meta = KVBatchMeta( - partition_id="canonical", - task_name=None, - sample_ids=list(request.canonical_sample_ids), - fields=None, - ) + client.buffer = ctrl._buffer + ctrl._buffer.set_data_plane_checkpoint_barrier(ctrl._data_plane_checkpoint_barrier) - async def _commit_then_discard() -> None: - buffer.reserve( - weight_version=3, - group_id=request.group_id, - rollout_ids=list(request.rollout_ids), - ) - async with ctrl._data_plane_checkpoint_barrier.mutation() as cut: - await buffer.commit_finalized(cut, request.group_id, meta, 3, 3) - assert buffer.size() == 1 - assert await ctrl._discard_committed_group(request) is True + async def _enrich(meta: KVBatchMeta, extra_env_info: Any) -> None: + enricher_calls.append(meta.sample_ids[0]) - asyncio.run(_commit_then_discard()) + ctrl._buffer.set_post_write_enricher(_enrich) + ctrl._buffer.reserve(weight_version=3, target_step=3, group_id="group") - assert buffer.size() == 0 + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) + assert outcome == _BelowValidFraction( + valid_count=1, + total_count=2, + replacement=None, + lineage_group_id=None, + reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + ) + assert ctrl._buffer.group_ids == () + assert client.ready_during_clear == [False, False] + assert client.clear_calls == [ + {"sample_ids": ["group_g0"], "partition_id": "canonical"}, + {"sample_ids": ["group_g0/call"], "partition_id": "staging"}, + ] + # Never committed: no enrichment, no pending metrics, and the step was + # credited short because no spare was left. + assert enricher_calls == [] + assert "group" not in ctrl._rollout_runtime.finalizer_metrics_by_group + assert "group" not in ctrl._rollout_runtime.finalizer_rewards_by_group + assert ctrl._rollout_runtime.batch_shortfall == {3: 1} + assert ctrl._available_finalizers.qsize() == 1 -def test_discard_yields_to_a_group_the_train_pump_already_claimed() -> None: - """Claiming removes the slot, after which remove_group raises for it. - Mutation cuts do not exclude one another, so the train pump can claim a ready group - while the discard waits on the barrier. Treating that ValueError as a cleanup - failure would turn a benign race into a fatal RuntimeError, and clearing the rows - anyway would pull them out from under a step that is already running. - """ - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize())) - ctrl._buffer.remove_group = AsyncMock( - side_effect=ValueError("unknown group_ids=['group']") +def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut() -> None: + """The rejected group's ledger discard and its spare's reservation share a cut.""" + spare = {"idx": 99} + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) ) - request = _request() - ctrl._rollout_runtime.finalizer_metrics_by_group["group"] = { - "finalize/invalid_row_rate": 0.0 - } - ctrl._rollout_runtime.finalizer_rewards_by_group["group"] = [] + _below_threshold(ctrl, spares=[spare]) + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" - assert asyncio.run(ctrl._discard_committed_group(request)) is False + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) - ctrl._buffer.abort.assert_not_called() - assert ctrl._dp_client.clear_calls == [], "rows belong to the claiming step" - # The claiming step consumes the group, so its post-step release still reaches - # these; popping them here would drop metrics the trainer is about to report. - assert "group" in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" in ctrl._rollout_runtime.finalizer_rewards_by_group + assert outcome == _BelowValidFraction( + valid_count=1, + total_count=2, + replacement=spare, + lineage_group_id="spare-group", + reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + ) + ctrl._buffer.commit_finalized.assert_not_awaited() + ctrl._buffer.abort.assert_called_once_with("group") + (discard_cut, discarded), _ = ctrl._rollout_recovery_ledger.discard_group.call_args + (reserve_cut, reserved), kwargs = ( + ctrl._rollout_manager.reserve_prompt_group.call_args + ) + assert discarded == "group" + assert reserved is spare + assert kwargs == {"target_step": 3} + assert reserve_cut is discard_cut + assert not ctrl._rollout_runtime.replacement_reserve + assert ctrl._rollout_runtime.batch_shortfall == {} def test_actor_rpc_failure_is_fatal_and_does_not_retry_or_requeue_actor() -> None: @@ -509,7 +565,11 @@ def test_actor_rpc_failure_is_fatal_and_does_not_retry_or_requeue_actor() -> Non request = _request() with pytest.raises(RuntimeError, match="actor died after submission"): - asyncio.run(ctrl._finalize_with_actor(request, {})) + asyncio.run( + ctrl._finalize_with_actor( + request, {}, target_step=None, replacements_used=0 + ) + ) assert finalize.calls == [request] assert ctrl._available_finalizers.empty() @@ -532,7 +592,11 @@ def test_missing_actor_metadata_cleans_known_canonical_and_staging_ownership() - ctrl = _controller(actor) with pytest.raises(RuntimeError, match="no metadata for non-dropped group"): - asyncio.run(ctrl._finalize_with_actor(_request(), {})) + asyncio.run( + ctrl._finalize_with_actor( + _request(), {}, target_step=None, replacements_used=0 + ) + ) assert ctrl._dp_client.clear_calls == [ {"sample_ids": ["group_g0"], "partition_id": "canonical"}, @@ -562,7 +626,9 @@ def test_dropped_actor_group_cleans_ownership_and_returns_uncommitted() -> None: actor = SimpleNamespace(finalize=_RemoteFinalize(result=result)) ctrl = _controller(actor) - committed = asyncio.run(ctrl._finalize_with_actor(_request(), {})) + committed = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=None, replacements_used=0) + ) assert committed is None assert ctrl._dp_client.clear_calls == [ diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index fa3f2bc7b53..85e3988a549 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -46,6 +46,7 @@ from nemo_rl.algorithms.single_controller import ( RolloutRuntimeState, SingleControllerActor, + _BelowValidFraction, ) from nemo_rl.algorithms.single_controller_utils.config import ( AsyncRLConfig, @@ -1476,7 +1477,6 @@ def test_min_valid_fraction_counts_rollouts_independently_of_segments( row_counts: tuple[int, int], threshold: float | None, forest_training: bool, - capsys: pytest.CaptureFixture[str], ) -> None: valid_rows, total_rows = row_counts # Three of four owners verified, regardless of how many rows they produced. @@ -1497,8 +1497,34 @@ def test_min_valid_fraction_counts_rollouts_independently_of_segments( else {} ), ) - fraction = 0.75 if forest_training else valid_rows / total_rows - dropped = threshold is not None and fraction < threshold + counts = (3, 4) if forest_training else row_counts + dropped = threshold is not None and counts[0] / counts[1] < threshold + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._master_config = SimpleNamespace( + token_capture=SimpleNamespace( + min_valid_fraction_per_group=threshold, forest_training=forest_training + ), + ) + + assert ctrl._valid_fraction_counts(finalized) == (counts if dropped else None) + + +@pytest.mark.parametrize("has_spare", [False, True]) +def test_below_valid_fraction_outcome_substitutes_or_closes_short( + has_spare: bool, capsys: pytest.CaptureFixture[str] +) -> None: + """The pump acts on the finalize cut's decision instead of re-deciding it.""" + spare = {"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1} + rejected = _BelowValidFraction( + valid_count=3, + total_count=4, + replacement=spare if has_spare else None, + lineage_group_id="spare-group" if has_spare else None, + reward_observations=[], + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) async def run() -> None: ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) @@ -1508,7 +1534,7 @@ async def run() -> None: ctrl._master_config = SimpleNamespace( grpo=GRPOConfig.model_construct(max_num_epochs=1), token_capture=SimpleNamespace( - min_valid_fraction_per_group=threshold, forest_training=forest_training + min_valid_fraction_per_group=0.8, forest_training=False ), ) ctrl._algo_cfg = ctrl._master_config.grpo @@ -1523,8 +1549,7 @@ async def run() -> None: ) _init_pump_ledgers(ctrl) ctrl._finalizer_actors = [object()] - ctrl._finalize_with_actor = AsyncMock(return_value=finalized) - ctrl._discard_committed_group = AsyncMock() + ctrl._finalize_with_actor = AsyncMock(side_effect=[rejected, committed]) ctrl._sampler = WindowedSampler(None, max_staleness_versions=1) ctrl._dataloader = [ BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) @@ -1541,20 +1566,22 @@ async def run() -> None: await ctrl._rollout_pump() - assert ctrl._rollout_manager.stats.committed == int(not dropped) - assert ctrl._buffer_capacity._value == int(dropped) + assert ctrl._rollout_manager.stats.committed == int(has_spare) + assert ctrl._buffer_capacity._value == int(not has_spare) assert ctrl._inflight_rollouts == 0 - assert ctrl._rollout_exhausted.is_set() - if dropped: - ctrl._discard_committed_group.assert_awaited_once_with(request) - ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_called_once() - else: - ctrl._discard_committed_group.assert_not_awaited() + generate = ctrl._rollout_manager.generate_for_finalization + if has_spare: + # The spare runs under the lineage reserved inside the finalize cut. + assert generate.await_count == 2 + assert generate.await_args.args == (spare,) + assert generate.await_args.kwargs["lineage_group_id"] == "spare-group" ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_not_called() + else: + assert generate.await_count == 1 + ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_called_once() asyncio.run(run()) - if dropped and forest_training: - assert "(3/4 < 0.8)" in capsys.readouterr().out + assert "(3/4 < 0.8)" in capsys.readouterr().out def test_actor_path_releases_generation_permit_before_finalization() -> None: @@ -1590,6 +1617,7 @@ async def _delayed_finalize( rollout_metrics: dict[str, Any], *, target_step: int | None = None, + replacements_used: int = 0, ) -> bool: nonlocal finalizers_started del request, rollout_metrics, target_step @@ -1762,6 +1790,7 @@ async def _main() -> None: async def _finalize( request: Any, rollout_metrics: dict[str, Any], + **_: Any, ) -> Any: async with ctrl._data_plane_checkpoint_barrier.mutation() as cut: manager.recovery_ledger.discard_group(cut, request.group_id) From 0f8a21eed20fe2870e41c9f32531790d44c1b64d Mon Sep 17 00:00:00 2001 From: Jonas Yang Date: Tue, 3 Nov 2026 22:46:40 +0800 Subject: [PATCH 2/8] test(single_controller): pin the below-threshold replacement contract Review injected three regressions the suite missed; each now fails: - the pump dropping target_step / replacements_used when it calls _finalize_with_actor (replacement and shortfall silently disabled), - the pump crediting a rejected step short a second time, - reserving the spare's lineage with rollout recovery disabled, which the pump would then never discard. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonas Yang --- .../test_finalizer_lifecycle.py | 31 +++++++++++++------ .../single_controller/test_rollout_pump.py | 13 +++++++- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index a01def6dd67..6443125c1cf 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -523,13 +523,21 @@ async def _enrich(meta: KVBatchMeta, extra_env_info: Any) -> None: assert ctrl._available_finalizers.qsize() == 1 -def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut() -> None: - """The rejected group's ledger discard and its spare's reservation share a cut.""" +@pytest.mark.parametrize("recovery", [True, False]) +def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut( + recovery: bool, +) -> None: + """The rejected group's ledger discard and its spare's reservation share a cut. + + Without rollout recovery the spare is not reserved here: the pump never + discards a lineage it did not get, so generate_for_finalization must own it. + """ spare = {"idx": 99} ctrl = _controller( SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) ) _below_threshold(ctrl, spares=[spare]) + ctrl._rollout_recovery_enabled = recovery ctrl._rollout_recovery_ledger.__contains__.return_value = True ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" @@ -541,19 +549,24 @@ def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut() -> None: valid_count=1, total_count=2, replacement=spare, - lineage_group_id="spare-group", + lineage_group_id="spare-group" if recovery else None, reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], ) ctrl._buffer.commit_finalized.assert_not_awaited() ctrl._buffer.abort.assert_called_once_with("group") + ctrl._rollout_recovery_ledger.discard_group.assert_called_once() (discard_cut, discarded), _ = ctrl._rollout_recovery_ledger.discard_group.call_args - (reserve_cut, reserved), kwargs = ( - ctrl._rollout_manager.reserve_prompt_group.call_args - ) assert discarded == "group" - assert reserved is spare - assert kwargs == {"target_step": 3} - assert reserve_cut is discard_cut + if recovery: + ctrl._rollout_manager.reserve_prompt_group.assert_called_once() + (reserve_cut, reserved), kwargs = ( + ctrl._rollout_manager.reserve_prompt_group.call_args + ) + assert reserved is spare + assert kwargs == {"target_step": 3} + assert reserve_cut is discard_cut + else: + ctrl._rollout_manager.reserve_prompt_group.assert_not_called() assert not ctrl._rollout_runtime.replacement_reserve assert ctrl._rollout_runtime.batch_shortfall == {} diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index 85e3988a549..4dd53149f78 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -1550,7 +1550,10 @@ async def run() -> None: _init_pump_ledgers(ctrl) ctrl._finalizer_actors = [object()] ctrl._finalize_with_actor = AsyncMock(side_effect=[rejected, committed]) - ctrl._sampler = WindowedSampler(None, max_staleness_versions=1) + # in_order stamps the batch for step 0, so the step the finalize cut + # replaces into (or credits short) is visible. + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) ctrl._dataloader = [ BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) ] @@ -1566,6 +1569,14 @@ async def run() -> None: await ctrl._rollout_pump() + # The finalize cut decides replacement and shortfall from these, so the + # pump must pass the stamped step and its replacement count. + assert [call.kwargs for call in ctrl._finalize_with_actor.await_args_list] == [ + {"target_step": 0, "replacements_used": 0} + ] + ([{"target_step": 0, "replacements_used": 1}] if has_spare else []) + # The finalize cut owns the shortfall credit; the pump adds none. + assert ctrl._rollout_runtime.batch_shortfall == {} + assert ctrl._rollout_exhausted.is_set() assert ctrl._rollout_manager.stats.committed == int(has_spare) assert ctrl._buffer_capacity._value == int(not has_spare) assert ctrl._inflight_rollouts == 0 From 6445f3a3e04dd195f11459d2d96428b5447c7b1a Mon Sep 17 00:00:00 2001 From: ruit Date: Thu, 12 Nov 2026 18:23:19 -0800 Subject: [PATCH 3/8] test(single_controller): pass finalize arguments explicitly in the trace test _finalize_with_actor now requires target_step and replacements_used so that a forgotten argument raises instead of silently disabling the replacement and shortfall handling. test_rollout_trace still called it with neither. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: ruit --- tests/unit/single_controller/test_rollout_trace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/single_controller/test_rollout_trace.py b/tests/unit/single_controller/test_rollout_trace.py index 9046e886499..8fcee59138d 100644 --- a/tests/unit/single_controller/test_rollout_trace.py +++ b/tests/unit/single_controller/test_rollout_trace.py @@ -482,7 +482,10 @@ def test_a_published_group_traces_each_call_and_its_version(tmp_path: Path) -> N # The v4 refit paused c1 after 6 of its 10 tokens. ctrl._refit_cuts.record(3, 4, [["c1", 6]]) - assert asyncio.run(ctrl._finalize_with_actor(_request(), {})) is result + finalized = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=None, replacements_used=0) + ) + assert finalized is result ctrl._rollout_trace.close() (published,) = [ From 7acbf2a6322cc7e959410c6109d46e85b7416b88 Mon Sep 17 00:00:00 2001 From: ruit Date: Thu, 12 Nov 2026 22:12:56 -0800 Subject: [PATCH 4/8] feat(sc): support dynamic sampling on the token-capture path On the capture path the rewards are final only after the finalizer has verified a group's rows, so the zero-variance filter runs in the finalize cut, before commit_finalized, next to the min_valid_fraction rejection introduced for the below-threshold drop. A rejected group is cleaned up like a structural finalizer drop and never becomes selectable; its replacement, the borrow from a later step's finished group, and the spare's ledger reservation share that cut, so no snapshot sees the group gone without its replacement. The filtered pool cannot be waited on inside the cut, so a finalizing group first claims a spare (_claim_filtered_spare). A claim is only a count against the pool; the spare is popped inside the cut, on rejection. Both rollout paths filter on the reward the environment returned, before effort shaping and reward penalties: Completion.reward_original on the ordinary path and FinalizedGroup.reward_original on the capture path. Which rows vote follows the masking policy on both: the finalizer hands back the raw mask_sample / truncated flags and loss_multiplier, and dynamic_sampling.voting_rewards_from_record and voting_rewards_from_finalized both delegate to voting_rewards. Training and metrics keep the trained reward. Setup no longer rejects token_capture.enabled together with grpo.use_dynamic_sampling. The controller-side finalize outcomes (RejectedGroup, BelowValidFractionGroup, ZeroVarianceGroup, and the FinalizeOutcome alias) live in single_controller_utils/utils.py next to the other controller result types. The three places that put a spare prompt in a dropped group's place (the below-valid and zero-variance rejections, and the pump's replacement of a skipped or filtered group) share _substitute_dropped_group. A below-valid capture group now borrows a finished group from a later step like the others; the borrow does not depend on the rollout path. A nightly runs the capture path with dynamic sampling, reward shaping and reward scaling on (dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated- async-gym-single-controller-token_capture). Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: ruit --- docs/guides/single-controller.md | 5 +- ...c-gym-single-controller-token_capture.yaml | 31 ++ ...n-vllm-noncolocated-single-controller.yaml | 5 - nemo_rl/algorithms/single_controller.py | 271 +++++++++++++----- .../single_controller_utils/config.py | 8 - .../single_controller_utils/utils.py | 49 +++- nemo_rl/experience/dynamic_sampling.py | 83 +++++- nemo_rl/experience/interfaces.py | 3 + nemo_rl/experience/rollout_manager.py | 13 +- nemo_rl/experience/rollout_reassembler.py | 25 ++ ...ync-gym-single-controller-token_capture.sh | 114 ++++++++ tests/test_suites/nightly.txt | 3 + .../algorithms/test_advantage_validity.py | 2 +- .../test_rollout_generation_failures.py | 6 +- tests/unit/experience/test_rollout_manager.py | 14 +- .../test_rollout_reassembler_actor.py | 4 + tests/unit/experience/test_rollouts.py | 9 +- .../test_selected_forest_boundaries.py | 6 + .../test_finalizer_lifecycle.py | 270 ++++++++++++++++- .../unit/single_controller/test_ppo_setup.py | 7 +- .../single_controller/test_rollout_pump.py | 261 ++++++++++++++++- .../test_step_reward_metrics.py | 74 ++++- tests/unit/test_effort_shaping.py | 2 + 23 files changed, 1125 insertions(+), 140 deletions(-) create mode 100644 examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml create mode 100755 tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index 779a6524480..0ad850130fa 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -327,14 +327,15 @@ Field definitions: ## Dynamic sampling -Set `grpo.use_dynamic_sampling: true` to drop prompt groups whose generations all received the same reward (zero within-group variance, so no advantage signal) and resample a substitute, as in [DAPO](dapo.md). A dropped group is always replaced, and the step it was stamped for is filled immediately from a later step's finished group where one exists, so the trainer does not wait out the resample. Setup requires: +Set `grpo.use_dynamic_sampling: true` to drop prompt groups whose generations all received the same reward (zero within-group variance, so no advantage signal) and resample a substitute, as in [DAPO](dapo.md). The filter reads the environment's reward as the environment returned it, before `env.nemo_gym.effort_levels` shaping and `reward_penalties`, on both rollout paths. A group whose rewards differ only because of that shaping or a penalty is therefore dropped, even though the trained rewards differ. A dropped group is always replaced, and the step it was stamped for is filled immediately from a later step's finished group where one exists, so the trainer does not wait out the resample. Setup requires: - `grpo.dynamic_sampling_max_gen_batches >= 1` (default `10`). A step may consume this many generation batches in total, the batch it was admitted with included, counted in prompts. Past that the run raises, as v1 does: a step that cannot be filled within its allowance means the filter is rejecting faster than the data can refill it. This is a separate budget from `async_rl.rollout_failure.max_replacement_attempts`, which covers rollout failures only; neither drop spends the other's allowance. Size it from the drop rate the run reports. At drop rate `p` a step needs about `1 / (1 - p)` prompts per committed group, so `grpo.dynamic_sampling_max_gen_batches` has to exceed `1 / (1 - p)` with room for the variance around it. Read `p` off `zero_variance_fraction`. - `async_rl.sampler.name: in_order`. A filtered group is charged to the target step the sampler stamps, and the spare pool is refilled while the pump waits to admit; no other built-in supports both, and a `custom` class cannot declare at setup that it does. - `grpo.num_generations_per_prompt >= 2`. -- `token_capture.enabled: false`. The capture path cannot borrow a finished group from a later step, so each drop would stall its step for a full fresh rollout; setup rejects the combination rather than paying that on every step. + +With `token_capture.enabled: true` the rewards are final only after the finalizer has verified the group's rows, so the filter runs there: a group whose voting rewards carry no signal is rejected before it is published, its rows are cleared, and it never becomes selectable. The replacement, the borrow and the per-step allowance work as above. Which rows vote follows `grpo.masked_reward_policy`, `grpo.overlong_filtering` and the environment's `mask_sample` flags, as on the non-capture path; a row that failed capture verification never votes. A filtered group has already paid for row assembly, so a high drop rate costs more here than without capture. `token_capture.min_valid_fraction_per_group` is checked first, and a group that fails it is replaced from the fault-tolerance pool (`async_rl.rollout_failure`), not the filtering pool. SC pre-warms its filtering pool with one dataloader batch (`grpo.num_prompts_per_step` raw prompts). Below that low-water mark it adds a whole batch, so the pool can hold more than PPS prompts. An empty pool is refilled on demand, including while admission or dispatch is blocked. This is an internal policy, independent of the fault-tolerance pool controlled by `async_rl.rollout_failure.replacement_reserve_prompts`; only `grpo.dynamic_sampling_max_gen_batches` limits the generation attempts for a training step. diff --git a/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml b/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml new file mode 100644 index 00000000000..338c090d481 --- /dev/null +++ b/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml @@ -0,0 +1,31 @@ +# DAPO-style sibling of the SingleController token-capture Gym nightly: dynamic +# sampling drops a group whose rollouts all score the same before it is +# finalized and replaces it from the filtering pool, and the reward is shaped +# and scaled as in the DAPO recipes. This is the nightly that exercises that +# rejection with real captured rows, including the cleanup of the staged rows and +# the spare's ledger reservation. +# Launch with: +# uv run examples/run_grpo_single_controller.py \ +# --config examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml +defaults: ./grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml + +grpo: + use_dynamic_sampling: true + # Map the 0/1 task reward to [-1, 1] before the overlong penalty is subtracted. + reward_scaling: + enabled: true + target_min: -1.0 + # Soft penalty on the last 512 tokens before the generation limit. + reward_shaping: + enabled: true + overlong_buffer_length: 512 + overlong_buffer_penalty: 1 + max_response_length: ${policy.generation.max_new_tokens} + +checkpointing: + checkpoint_dir: results/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture + +logger: + log_dir: logs/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture + wandb: + name: dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture diff --git a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml index da26a4cf1ab..a15210ffe6b 100644 --- a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml +++ b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml @@ -5,11 +5,6 @@ # No NeMo-Gym: the filter is a property of the rollout pump, not of the # environment, so the cheaper built-in math env exercises it just as well and # needs no dataset preparation step in CI. -# -# Non-capture on purpose. Setup rejects dynamic sampling together with -# token_capture.enabled, because the capture path cannot borrow a finished -# group from a later step and every drop would stall its step for a fresh -# rollout. # Launch with: # uv run examples/run_grpo_single_controller.py \ # --config examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml diff --git a/nemo_rl/algorithms/single_controller.py b/nemo_rl/algorithms/single_controller.py index 6b48ab9ceca..8982e8c937f 100644 --- a/nemo_rl/algorithms/single_controller.py +++ b/nemo_rl/algorithms/single_controller.py @@ -52,7 +52,7 @@ import warnings from collections import deque from collections.abc import AsyncIterator, Iterator -from dataclasses import dataclass, replace +from dataclasses import replace from functools import partial from pathlib import Path from typing import ( @@ -147,6 +147,10 @@ _register_single_controller_partitions, ) from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + FinalizeOutcome, + RejectedGroup, + ZeroVarianceGroup, aggregate_step_metrics, reduce_advantage_pump_metrics, ) @@ -178,7 +182,10 @@ EnvironmentCheckpoint, ) from nemo_rl.environments.nemo_gym import should_use_nemo_gym -from nemo_rl.experience.dynamic_sampling import rewards_carry_no_signal +from nemo_rl.experience.dynamic_sampling import ( + rewards_carry_no_signal, + voting_rewards_from_finalized, +) from nemo_rl.experience.failures import RolloutStall from nemo_rl.experience.payload import VIOLATION_TAG_KEYS from nemo_rl.experience.reward_penalties import ( @@ -250,24 +257,6 @@ _SUPERVISOR_DRAIN_TIMEOUT_S = 30.0 -@dataclass(frozen=True) -class _BelowValidFraction: - """A finalized group rejected by ``min_valid_fraction_per_group`` before commit. - - The group never became selectable. ``replacement`` is the spare prompt taken - in the same mutation cut that cleaned the group up (None when the step was - credited short instead), and ``lineage_group_id`` is its ledger reservation - when rollout recovery is enabled. ``reward_observations`` are the valid rows' - rewards, which the pump still tallies into the step's reward metrics. - """ - - valid_count: int - total_count: int - replacement: Optional[DatumSpec] - lineage_group_id: Optional[str] - reward_observations: list[FinalizedReward] - - def _latest_generation_values( metrics: dict[str, Any], metric_name: str ) -> dict[int, float]: @@ -787,6 +776,8 @@ def __init__( ) # Prompts a step has drawn beyond the batch it was admitted with. self._filtered_prompts_by_step: dict[int, int] = {} + # Filtered spares promised to groups that are still finalizing. + self._filtered_spares_claimed: int = 0 # Set once the dataloader has nothing left, so waiters stop waiting. self._dataloader_exhausted: bool = False @@ -1369,6 +1360,51 @@ def _valid_fraction_counts( return valid_count, total_count return None + def _voting_rewards(self, finalized: "FinalizedGroup") -> list[float]: + """The group's rewards that vote in its baseline under the masking policy.""" + return voting_rewards_from_finalized( + finalized, + masked_reward_policy=( + None + if self._is_ppo + else cast(GRPOConfig, self._algo_cfg).masked_reward_policy + ), + overlong_filtering=self._algo_cfg.overlong_filtering, + ) + + @contextlib.asynccontextmanager + async def _claim_filtered_spare(self) -> AsyncIterator[None]: + """Promise a finalizing group a filtered spare, in case it is rejected. + + A rejected group's spare is taken inside the finalize cut, which cannot wait + for the pump to refill an empty pool. Claiming first waits until the pool + holds a spare no other finalizing group has claimed. A claim only counts + against the pool; the spare is popped inside the cut, and only on rejection. + + No claim is made once the dataloader is exhausted: nothing will refill the + pool, so a rejected group takes whatever is left or closes its step short. + """ + if not self._dynamic_sampling_enabled: + yield + return + claimed = False + while True: + # No await between the check and the claim, so two groups cannot claim one spare. + if len(self._rollout_runtime.filtered_reserve) > ( + self._filtered_spares_claimed + ): + self._filtered_spares_claimed += 1 + claimed = True + break + if self._dataloader_exhausted: + break + await asyncio.sleep(0.1) + try: + yield + finally: + if claimed: + self._filtered_spares_claimed -= 1 + async def _finalize_with_actor( self, request: "ReassemblyRequest", @@ -1376,7 +1412,7 @@ async def _finalize_with_actor( *, target_step: Optional[int], replacements_used: int, - ) -> Union["FinalizedGroup", _BelowValidFraction, None]: + ) -> FinalizeOutcome: """Finalize and index one group atomically with respect to TQ saves. Returns the committed FinalizedGroup once the group is committed to @@ -1384,13 +1420,17 @@ async def _finalize_with_actor( structural outcome (ownership already cleaned up; the caller credits the step short). - A group below ``token_capture.min_valid_fraction_per_group`` is + A group below ``token_capture.min_valid_fraction_per_group``, or with + dynamic sampling on one whose voting rewards carry no signal, is rejected before ``commit_finalized`` marks its slot ready, so the sampler can never select it. Its cleanup, the spare prompt that replaces it (or the step's shortfall credit), and that spare's ledger reservation share the commit's mutation cut, so no snapshot sees the - group gone without its replacement. Returns a ``_BelowValidFraction`` - in that case. + group gone without its replacement. Returns a ``BelowValidFractionGroup`` + or ``ZeroVarianceGroup`` in that case. A zero-variance group takes its + spare from the filtered pool, which cannot be waited on inside the cut: + the caller holds ``_claim_filtered_spare`` across this call so the pool + is not empty when the group is rejected. """ self._finalizer_waiters += 1 queue_depth = max( @@ -1411,7 +1451,7 @@ async def _finalize_with_actor( finalize_start = time.perf_counter() rpc_submitted = False actor_reusable = False - rejected: Optional[_BelowValidFraction] = None + rejected: Optional[RejectedGroup] = None try: # The actor publishes canonical rows before returning metadata. Keep # the remote write, local replay-index update, and lineage hand-off in @@ -1441,6 +1481,9 @@ async def _finalize_with_actor( else: actor_reusable = True + zero_variance = not finalized.dropped and rewards_carry_no_signal( + self._voting_rewards(finalized) + ) if finalized.dropped: try: await self._cleanup_known_finalization_request_unlocked( @@ -1483,19 +1526,39 @@ async def _finalize_with_actor( f"failed for group {request.group_id}" ) from cleanup_error replacement = self._take_replacement(replacements_used) - lineage_group_id = None - if replacement is None: - self._credit_shortfall(target_step) - elif self._rollout_recovery_enabled: - lineage_group_id = self._rollout_manager.reserve_prompt_group( - cut, replacement, target_step=target_step - ) - rejected = _BelowValidFraction( + lineage_group_id, replacement_step = self._substitute_dropped_group( + cut, replacement, target_step + ) + rejected = BelowValidFractionGroup( + replacement=replacement, + lineage_group_id=lineage_group_id, + target_step=replacement_step, + reward_observations=list(finalized.reward_observations), + zero_variance=zero_variance, valid_count=counts[0], total_count=counts[1], + ) + committed = False + elif self._dynamic_sampling_enabled and zero_variance: + try: + await self._cleanup_known_finalization_request_unlocked( + cut, request + ) + except BaseException as cleanup_error: + raise RuntimeError( + "finalizer group carried no reward signal and known-key " + f"cleanup failed for group {request.group_id}" + ) from cleanup_error + replacement = self._pop_filtered_replacement(target_step) + lineage_group_id, replacement_step = self._substitute_dropped_group( + cut, replacement, target_step + ) + rejected = ZeroVarianceGroup( replacement=replacement, lineage_group_id=lineage_group_id, + target_step=replacement_step, reward_observations=list(finalized.reward_observations), + zero_variance=True, ) committed = False else: @@ -1804,12 +1867,13 @@ async def _dispatch_one_prompt( self._credit_shortfall(target_step) return request, rollout_metrics = captured - finalized = await self._finalize_with_actor( - request, - rollout_metrics, - target_step=target_step, - replacements_used=replacements, - ) + async with self._claim_filtered_spare(): + finalized = await self._finalize_with_actor( + request, + rollout_metrics, + target_step=target_step, + replacements_used=replacements, + ) if finalized is None: # Finalizer dropped the group as a structural # outcome (e.g. router replay with no routed @@ -1820,42 +1884,63 @@ async def _dispatch_one_prompt( self._buffer_capacity.release() self._credit_shortfall(target_step) return - # No per-row masking flags here, so this count is metric-only. - capture_rewards = [ - row.reward for row in finalized.reward_observations - ] self._rollout_manager.record_group_rewards( target_step, - capture_rewards, - replacement_index=replacements, - zero_variance=rewards_carry_no_signal(capture_rewards), + [row.reward for row in finalized.reward_observations], + replacement_index=replacements + filtered_replacements, + zero_variance=( + finalized.zero_variance + if isinstance(finalized, RejectedGroup) + else rewards_carry_no_signal( + self._voting_rewards(finalized) + ) + ), ) - if not isinstance(finalized, _BelowValidFraction): + if not isinstance(finalized, RejectedGroup): self._rollout_manager.stats.committed += 1 ownership_transferred = True break - # Too few rollouts verified to be worth training on. # The finalize cut already cleaned the group up and # took its replacement (or credited the step short). - print( - f" finalize: group {request.group_id} below " - "min_valid_fraction_per_group " - f"({finalized.valid_count}/{finalized.total_count} < " - f"{self._master_config.token_capture.min_valid_fraction_per_group}); " - + ( - "substituting a spare prompt" - if finalized.replacement is not None - else "no replacement; dropping the group" - ), - flush=True, - ) - if finalized.replacement is None: - self._rollout_manager.record_finalizer_dropped_prompt() - self._buffer_capacity.release() - return - replacements += 1 + if isinstance(finalized, ZeroVarianceGroup): + self._rollout_manager.stats.zero_variance_groups += 1 + print( + f" dynamic sampling: dropped group " + f"{request.group_id} (target_step={target_step}), " + "its voting rewards carry no signal; " + + ( + "substituting a spare prompt" + if finalized.replacement is not None + else "no replacement; closing the step short" + ), + flush=True, + ) + if finalized.replacement is None: + self._buffer_capacity.release() + return + filtered_replacements += 1 + else: + # Too few rollouts verified to be worth training on. + print( + f" finalize: group {request.group_id} below " + "min_valid_fraction_per_group " + f"({finalized.valid_count}/{finalized.total_count} < " + f"{self._master_config.token_capture.min_valid_fraction_per_group}); " + + ( + "substituting a spare prompt" + if finalized.replacement is not None + else "no replacement; dropping the group" + ), + flush=True, + ) + if finalized.replacement is None: + self._rollout_manager.record_finalizer_dropped_prompt() + self._buffer_capacity.release() + return + replacements += 1 prompt = finalized.replacement lineage_group_id = finalized.lineage_group_id + target_step = finalized.target_step # A substitution is a fresh rollout, not a # continuation of the one that fell short, so it # observes the same pause a first dispatch does @@ -1946,19 +2031,11 @@ async def _dispatch_one_prompt( # a replacement, or crediting the target step short must be # one checkpoint-atomic decision. self._discard_prompt_group(cut, lineage_group_id) - if replacement is not None: - # Attempted only now that a spare is in hand, because - # the borrow is a debt and the spare is what repays it. - # Borrowing without one would leave the lender short - # instead: the same hole, one step later. - lender_step = self._promote_into_step(target_step) - if lender_step is not None: - target_step = lender_step - lineage_group_id = self._reserve_prompt_group( - cut, replacement, target_step=target_step + lineage_group_id, target_step = ( + self._substitute_dropped_group( + cut, replacement, target_step ) - else: - self._credit_shortfall(target_step) + ) if replacement is None: # Nothing was committed, so the train pump will never see this # group and never release its permit on our behalf. @@ -2483,6 +2560,46 @@ def _promote_into_step(self, target_step: Optional[int]) -> Optional[int]: ) return lender_step + def _substitute_dropped_group( + self, + cut: Optional[DataPlaneMutationCut], + replacement: Optional[DatumSpec], + target_step: Optional[int], + ) -> tuple[Optional[str], Optional[int]]: + """Stand a spare prompt in for a dropped group, or close its step short. + + Call inside the cut that removed the dropped group, so no snapshot sees the + group gone without its replacement. A finished group from a later step fills + the dropped step when one exists, and the spare is then generated for that + lender. + + Args: + cut: The cut the group was removed in; ``None`` when nothing is recorded. + replacement: The spare already taken from its pool, or ``None`` if there + was none, which credits ``target_step`` short. + target_step: The step the dropped group was stamped for. + + Returns: + The ledger ID reserved for the spare (``None`` without rollout recovery + or without a spare) and the step the spare is generated for. + """ + if replacement is None: + self._credit_shortfall(target_step) + return None, target_step + # Attempted only now that a spare is in hand, because the borrow is a debt + # and the spare is what repays it. Borrowing without one would leave the + # lender short instead: the same hole, one step later. + lender_step = self._promote_into_step(target_step) + if lender_step is not None: + target_step = lender_step + lineage_group_id = None + if self._rollout_recovery_enabled: + assert cut is not None + lineage_group_id = self._rollout_manager.reserve_prompt_group( + cut, replacement, target_step=target_step + ) + return lineage_group_id, target_step + def _credit_shortfall(self, target_step: Optional[int]) -> None: """Record that a stamped step will never receive a group it is waiting for.""" if target_step is None: diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index b64a0ff4373..cb9c6e58699 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -1552,14 +1552,6 @@ def _validate_algo_settings(master_config: MasterConfig) -> None: f"grpo.use_dynamic_sampling requires grpo.{key} to be 0 or null: " "nonzero message-level overrides can train constant-reward groups." ) - # Capture cannot borrow future groups, so routine filtering would stall training. - if master_config.token_capture.enabled: - raise ValueError( - f"{algo_prefix}.use_dynamic_sampling is not supported together with " - "token_capture.enabled: on the capture path a replacement cannot borrow " - "a finished group from a later step, so every dropped group stalls its " - "step for a full fresh rollout. Disable one of the two." - ) if grpo_cfg.dynamic_sampling_max_gen_batches < 1: raise ValueError( f"{algo_prefix}.dynamic_sampling_max_gen_batches must be >= 1 when " diff --git a/nemo_rl/algorithms/single_controller_utils/utils.py b/nemo_rl/algorithms/single_controller_utils/utils.py index b3d73d18c9b..6539ce2a27c 100644 --- a/nemo_rl/algorithms/single_controller_utils/utils.py +++ b/nemo_rl/algorithms/single_controller_utils/utils.py @@ -17,7 +17,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any, Optional import numpy as np import torch @@ -26,6 +26,11 @@ from nemo_rl.algorithms.metric_utils import REWARD_KEY from nemo_rl.data_plane import KVBatchMeta +if TYPE_CHECKING: + from nemo_rl.data.interfaces import DatumSpec + from nemo_rl.experience.reward_penalties import FinalizedReward + from nemo_rl.experience.rollout_reassembler import FinalizedGroup + # Reduction rules for all_mb_metrics. Mirror grpo.py / grpo_sync.py. _MB_METRIC_MIN: frozenset[str] = frozenset( {"probs_ratio_min", "probs_ratio_clamped_min", "opd_full_reverse_kl_min"} @@ -177,6 +182,48 @@ def from_values(cls, values: torch.Tensor) -> AdvantagePartial: ) +@dataclass(frozen=True) +class RejectedGroup: + """A finalized group the controller rejected before commit. + + The group never became selectable. ``replacement`` is the spare prompt taken + in the same mutation cut that cleaned the group up (None when the step was + credited short instead), ``target_step`` the step it was reserved for, and + ``lineage_group_id`` its ledger reservation when rollout recovery is enabled. + ``reward_observations`` are the valid rows' rewards and ``zero_variance`` + whether the rows that vote carried no signal; the pump still tallies both + into the step's reward metrics. + """ + + replacement: Optional[DatumSpec] + lineage_group_id: Optional[str] + target_step: Optional[int] + reward_observations: list[FinalizedReward] + zero_variance: bool + + +@dataclass(frozen=True) +class BelowValidFractionGroup(RejectedGroup): + """Rejected by ``token_capture.min_valid_fraction_per_group``.""" + + valid_count: int + total_count: int + + +@dataclass(frozen=True) +class ZeroVarianceGroup(RejectedGroup): + """Rejected by dynamic sampling: the voting rewards left every row zero advantage. + + ``target_step`` is the lender's when the dropped step was filled from a later + step's finished group, which the replacement then repays. + """ + + +type FinalizeOutcome = FinalizedGroup | RejectedGroup | None +"""What ``_finalize_with_actor`` returns: the group committed, a group the controller +rejected, or ``None`` when the finalizer dropped it as a structural outcome.""" + + def reduce_advantage_pump_metrics( reward_partials: list[RewardPartial], advantage_partials: list[AdvantagePartial], diff --git a/nemo_rl/experience/dynamic_sampling.py b/nemo_rl/experience/dynamic_sampling.py index 739986fa540..b2490294996 100644 --- a/nemo_rl/experience/dynamic_sampling.py +++ b/nemo_rl/experience/dynamic_sampling.py @@ -16,11 +16,14 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from nemo_rl.data_plane.schema import MASK_SAMPLE from nemo_rl.experience.interfaces import PromptGroupRecord +if TYPE_CHECKING: + from nemo_rl.experience.rollout_reassembler import FinalizedGroup + def mask_sample_flagged(extra: dict[str, Any] | None) -> bool: """Read both mask flags, since Gym may default the first-class flag to false.""" @@ -32,9 +35,9 @@ def mask_sample_flagged(extra: dict[str, Any] | None) -> bool: def rewards_carry_no_signal(rewards: Sequence[float]) -> bool: - """Whether a group's participating rewards leave every row zero advantage. + """Whether a group's voting rewards leave every row zero advantage. - True for at most one participating reward, or when they are all exactly equal. + True for at most one voting reward, or when they are all exactly equal. Exact equality rather than a std threshold because float roundoff can give a uniform group a tiny nonzero std. @@ -47,13 +50,16 @@ def rewards_carry_no_signal(rewards: Sequence[float]) -> bool: return len(rewards) <= 1 or max(rewards) == min(rewards) -def participating_rewards( - record: PromptGroupRecord, +def voting_rewards( + rewards: Sequence[float], *, + mask_sample: Sequence[bool], + truncated: Sequence[bool], + loss_multiplier: float, masked_reward_policy: Optional[str], overlong_filtering: bool, ) -> list[float]: - """The rewards that vote in this group's baseline and std. + """The rewards that vote in a group's baseline and std. Mirrors v1's ``_dynamic_sampling_valid_mask``, which is the mask the group statistics are actually computed under. Filtering on the full reward list instead @@ -70,20 +76,71 @@ def participating_rewards( ``overlong_filtering`` so do truncated ones -- the rows the loss drops later in the step. Under ``include`` both still vote: their rewards are measurements of the policy even though they carry no loss. + + Args: + rewards: One Gym reward per row, before effort shaping and reward penalties. + mask_sample: Per-row environment ``mask_sample`` flag, parallel to ``rewards``. + truncated: Per-row truncation flag, parallel to ``rewards``. + loss_multiplier: The group's dataset loss weight. + masked_reward_policy: ``grpo.masked_reward_policy``; ``None`` is PPO, which + has no group-relative baseline to mask rows out of. + overlong_filtering: ``grpo.overlong_filtering``. """ - if record.loss_multiplier == 0: + if loss_multiplier == 0: return [] - # None is PPO: no group-relative baseline exists to mask rows out of. if masked_reward_policy != "exclude": - return [completion.reward for completion in record.completions] + return list(rewards) return [ - completion.reward - for completion in record.completions - if not mask_sample_flagged(completion.env_extras) - and not (overlong_filtering and completion.truncated) + reward + for reward, masked, is_truncated in zip( + rewards, mask_sample, truncated, strict=True + ) + if not masked and not (overlong_filtering and is_truncated) ] +def voting_rewards_from_record( + record: PromptGroupRecord, + *, + masked_reward_policy: Optional[str], + overlong_filtering: bool, +) -> list[float]: + """The rewards that vote in a group's baseline, read from its ``Completion`` rows.""" + return voting_rewards( + [ + completion.reward + if completion.reward_original is None + else completion.reward_original + for completion in record.completions + ], + mask_sample=[ + mask_sample_flagged(completion.env_extras) + for completion in record.completions + ], + truncated=[completion.truncated for completion in record.completions], + loss_multiplier=record.loss_multiplier, + masked_reward_policy=masked_reward_policy, + overlong_filtering=overlong_filtering, + ) + + +def voting_rewards_from_finalized( + finalized: "FinalizedGroup", + *, + masked_reward_policy: Optional[str], + overlong_filtering: bool, +) -> list[float]: + """The rewards that vote in a group's baseline, read from a finalized capture group.""" + return voting_rewards( + finalized.reward_original, + mask_sample=finalized.reward_mask_sample, + truncated=finalized.reward_truncated, + loss_multiplier=finalized.loss_multiplier, + masked_reward_policy=masked_reward_policy, + overlong_filtering=overlong_filtering, + ) + + @dataclass class StepRewardStats: """Reward totals for the groups generated for one training step. diff --git a/nemo_rl/experience/interfaces.py b/nemo_rl/experience/interfaces.py index 29320e93f2b..72751edf0fa 100644 --- a/nemo_rl/experience/interfaces.py +++ b/nemo_rl/experience/interfaces.py @@ -49,6 +49,9 @@ class Completion: env_extras: Optional[dict[str, Any]] truncated: bool reward: float + # The environment's reward before effort shaping and reward penalties, which + # dynamic sampling filters on. ``None`` means nothing rewrote ``reward``. + reward_original: Optional[float] = None @dataclass diff --git a/nemo_rl/experience/rollout_manager.py b/nemo_rl/experience/rollout_manager.py index 8f0168f20f1..2e4165f6ba2 100644 --- a/nemo_rl/experience/rollout_manager.py +++ b/nemo_rl/experience/rollout_manager.py @@ -52,9 +52,9 @@ StepRewardStats, drain_step_reward_metrics, mask_sample_flagged, - participating_rewards, record_group_rewards, rewards_carry_no_signal, + voting_rewards_from_record, ) from nemo_rl.experience.failures import ( FailureClass, @@ -1534,6 +1534,8 @@ async def consume(pending_rows: list[dict], refusal_streak: int = 0) -> None: else None, ) else: + # Effort shaping and the penalties rewrite this reward in place. + result["reward_original"] = float(result["full_result"]["reward"]) shaping_by_rowidx[rowidx] = _apply_effort_shaping( [result], [inputs_by_rowidx[rowidx]], self._effort_config ) @@ -1926,6 +1928,7 @@ def completion_extras(full_result: dict) -> dict: env_extras=env_extras, truncated=truncated, reward=float(result["full_result"]["reward"]), + reward_original=result.get("reward_original"), ) ) return completions, penalty_counts @@ -2500,9 +2503,9 @@ async def run_rollout( dispatch_recorder=dispatch_recorder, ) - def _participating_rewards(self, record: PromptGroupRecord) -> list[float]: + def _voting_rewards(self, record: PromptGroupRecord) -> list[float]: """This group's voting rewards under the run's masking policy.""" - return participating_rewards( + return voting_rewards_from_record( record, masked_reward_policy=self._masked_reward_policy, overlong_filtering=self._overlong_filtering, @@ -2510,7 +2513,7 @@ def _participating_rewards(self, record: PromptGroupRecord) -> list[float]: def _is_zero_variance_group(self, record: PromptGroupRecord) -> bool: """Whether this group's voters leave every row zero advantage.""" - return rewards_carry_no_signal(self._participating_rewards(record)) + return rewards_carry_no_signal(self._voting_rewards(record)) def record_group_rewards( self, @@ -2676,7 +2679,7 @@ async def generate_and_push( self._consecutive_infra_drops = 0 self._stats.zero_variance_groups += 1 rewards = [c.reward for c in record.completions] - voting = self._participating_rewards(record) + voting = self._voting_rewards(record) self.record_group_rewards( target_step, rewards, diff --git a/nemo_rl/experience/rollout_reassembler.py b/nemo_rl/experience/rollout_reassembler.py index 97cef6b5098..1d9c7b66e97 100644 --- a/nemo_rl/experience/rollout_reassembler.py +++ b/nemo_rl/experience/rollout_reassembler.py @@ -196,6 +196,14 @@ class FinalizedGroup: canonical_output_tokens: int = 0 metrics: dict[str, float] = field(default_factory=dict) reward_observations: list[FinalizedReward] = field(default_factory=list) + # Parallel to ``reward_observations``: each row's environment ``mask_sample`` + # flag, truncation and original reward (before effort shaping and penalties), plus + # the group's dataset loss weight. Which rewards vote in the group baseline + # depends on controller-side policy, so the raw flags travel instead of a verdict. + reward_mask_sample: list[bool] = field(default_factory=list) + reward_truncated: list[bool] = field(default_factory=list) + reward_original: list[float] = field(default_factory=list) + loss_multiplier: float = 1.0 # True when the finalizer rejected the whole group as a structural outcome # (see drop_reason); the caller aborts the slot instead of committing it. # Policy decisions like a low valid-rollout fraction are made by the caller, @@ -1092,6 +1100,13 @@ def finalize_group( # termination rates like the penalty counts; receipt-side rates only # saw a placeholder truncated=False. truncated_flags = self._truncated_flags(rows, logical) + reward_mask_sample = [ + mask_sample[index] for index, _, row in reward_rows if row.valid + ] + reward_truncated = [ + truncated_flags[index] for index, _, row in reward_rows if row.valid + ] + reward_original = [rewards[slot] for _, slot, row in reward_rows if row.valid] metrics["finalize/truncated_count"] = 0.0 for index, slot, row in reward_rows: if row.valid: @@ -1121,6 +1136,9 @@ def finalize_group( metrics=metrics, rollouts_ms=_rollouts_ms, reward_observations=reward_observations, + reward_mask_sample=reward_mask_sample, + reward_truncated=reward_truncated, + reward_original=reward_original, logical=logical, execution_row_multiple=execution_row_multiple, canonical_sample_ids=canonical_sample_ids, @@ -1154,6 +1172,9 @@ def _publish_rows( metrics: dict[str, float], rollouts_ms: float, reward_observations: list[FinalizedReward], + reward_mask_sample: list[bool], + reward_truncated: list[bool], + reward_original: list[float], logical: Optional[_LogicalRows], execution_row_multiple: int, canonical_sample_ids: list[str], @@ -1425,6 +1446,10 @@ def _publish_rows( ), metrics=metrics, reward_observations=reward_observations, + reward_mask_sample=reward_mask_sample, + reward_truncated=reward_truncated, + reward_original=reward_original, + loss_multiplier=loss_multiplier, valid_row_count=len(valid_rows), total_row_count=real_row_count, ) diff --git a/tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh b/tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh new file mode 100755 index 00000000000..09f4ef91d79 --- /dev/null +++ b/tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=2 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=3 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +# ~25 min startup (30B-MoE load + CUDA-graph warmup + nemo_gym servers) plus +# ~16 min/step, and dynamic sampling generates extra groups to replace the +# filtered ones. Three steps still cover a refit with requests in flight and a +# checkpoint save through the finalizer. +NUM_MINUTES=120 # Initial budget; calibrate against a completed run. +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +cd $PROJECT_ROOT + +# Prepare nemo-gym workplace_assistant dataset (mirrors tests/functional/grpo_async_gym.sh). +DATA_DIR=$EXP_DIR/data +mkdir -p $DATA_DIR +cd 3rdparty/Gym-workspace/Gym +if [[ ! -f env.yaml ]]; then + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "[ERROR] HF_TOKEN is not set" + exit 1 + fi + echo "hf_token: $HF_TOKEN" >> env.yaml +fi +uv run ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml]" \ + +output_dirpath=data/workplace_assistant \ + +mode=train_preparation \ + +should_download=true \ + +data_source=huggingface +cd - + +# Trim to a single tool so first prompt fits. +TRAIN_PATH=$DATA_DIR/workplace_assistant_train.jsonl +VALIDATION_PATH=$DATA_DIR/workplace_assistant_validation.jsonl +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/train.jsonl > $TRAIN_PATH +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/validation.jsonl > $VALIDATION_PATH + +# Token capture is only wired up on the SingleController path. +uv run examples/run_grpo_single_controller.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + checkpointing.save_period=3 \ + data.train.data_path=$TRAIN_PATH \ + data.validation.data_path=$VALIDATION_PATH \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Gates follow the non-capture DAPO single-controller dynamic-sampling nightly +# (dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.sh, #4588), +# plus the capture-only finalize/* health gates this recipe adds. +# +# The dynamic-sampling gates: the filter has to be doing something (>=1 zero-variance +# group filtered across the run) and has to be paying for it without shrinking the step +# (skipped_step and dropped_prompt_groups stay 0 -- the batch size the feature exists to +# hold). unfiltered_reward, not reward, is the progress signal: reward here is the mean +# over the trained (kept) rows, which excludes the all-same-reward groups, so it is the +# wrong denominator to gate progress on (v1 reports the two under swapped names; see +# docs/guides/single-controller.md#dynamic-sampling). +# +# The capture gates: a nonzero finalize/invalid_row_rate or capture_poisoned_rollouts +# means capture or recovery lost a rollout's tokens. +# +# Thresholds are for this recipe's scale (3 steps, 2n8g 30B, Megatron generation): +# total_step_time matches the capture nightly (<1000) plus headroom for the extra +# replacement generation dynamic sampling drives; no max(total_step_time) gate because +# save_period=3 bundles a checkpoint write into the final step. zero_variance >=1 and +# unfiltered_reward >0 are intentionally loose pending calibration against a completed run. +uv run tests/check_metrics.py $JSON_METRICS \ + "len(data[\"train/loss\"]) == $MAX_STEPS" \ + 'all_finite(data["train/loss"])' \ + 'all_finite(data["train/token_mult_prob_error"])' \ + 'median(data["train/token_mult_prob_error"]) < 1.02' \ + 'max(data["train/token_mult_prob_error"]) < 1.025' \ + 'median(data["train/unfiltered_reward"]) > 0.0' \ + 'median(data["timing/train/total_step_time"]) < 1500' \ + 'max(data["train/finalize/invalid_row_rate"]) == 0' \ + 'max(data["train/finalize/capture_poisoned_rollouts"]) == 0' \ + 'max(data["rollout/zero_variance_groups_total"]) >= 1' \ + 'min(data["train/skipped_step"]) == max(data["train/skipped_step"]) == 0' \ + 'min(data["train/dropped_prompt_groups"]) == max(data["train/dropped_prompt_groups"]) == 0' + +# Clean up checkpoint directory after successful run to save space. +rm -rf "$CKPT_DIR" diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index ec4c41c06cb..2d6e9c357dc 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -152,6 +152,9 @@ tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-a # nightly that exercises Megatron-inference capture (refits with requests in # flight, the chained-prefix cache under load) and gates on finalizer health. tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh +# The DAPO variant of the capture nightly above: dynamic sampling, reward shaping and +# reward scaling on, so zero-variance groups are rejected before they are finalized. +tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-noncolocated-async-gym-single-controller-token_capture-r3-defer.sh # Nano-v3.5 diff --git a/tests/unit/algorithms/test_advantage_validity.py b/tests/unit/algorithms/test_advantage_validity.py index 1d7724a508d..eed66044604 100644 --- a/tests/unit/algorithms/test_advantage_validity.py +++ b/tests/unit/algorithms/test_advantage_validity.py @@ -187,7 +187,7 @@ def test_reinforce_pp_none_valid_mask_keeps_legacy_all_valid_behavior(): @pytest.mark.parametrize("include", [False, True]) @pytest.mark.parametrize("leave_one_out", [False, True]) @pytest.mark.parametrize("normalize", [False, True]) -def test_gdpo_statistics_match_participating_rewards(include, leave_one_out, normalize): +def test_gdpo_statistics_match_voting_rewards(include, leave_one_out, normalize): """Check actual final values, including weighted components and whitening.""" prompt_ids = torch.tensor([[0]] * 4 + [[1]] * 4) loss_weights = torch.tensor([1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0]) diff --git a/tests/unit/experience/test_rollout_generation_failures.py b/tests/unit/experience/test_rollout_generation_failures.py index 3cf0f9bd308..adf94926e48 100644 --- a/tests/unit/experience/test_rollout_generation_failures.py +++ b/tests/unit/experience/test_rollout_generation_failures.py @@ -536,7 +536,11 @@ async def _result(rowidx): return ( rowidx, {"name": "agent"}, - {"input_message_log": [], "message_log": []}, + { + "input_message_log": [], + "message_log": [], + "full_result": {"reward": 1.0}, + }, None, ) diff --git a/tests/unit/experience/test_rollout_manager.py b/tests/unit/experience/test_rollout_manager.py index f7c5825e2f7..807c105fa0a 100644 --- a/tests/unit/experience/test_rollout_manager.py +++ b/tests/unit/experience/test_rollout_manager.py @@ -1458,7 +1458,7 @@ def _reward_penalty_result(output, assistant_overrides=None, assistant_tokens=No ), ], ) -def test_nemo_gym_reward_penalties_match_legacy_rewards_counts_and_metrics( +def test_nemo_reward_original_penalties_match_legacy_rewards_counts_and_metrics( reward_penalty_config, output, assistant_overrides, @@ -1468,16 +1468,19 @@ def test_nemo_gym_reward_penalties_match_legacy_rewards_counts_and_metrics( ): impl = _nemo_gym_impl(True, reward_penalty_config) result = _reward_penalty_result(output, assistant_overrides, assistant_tokens) + result["reward_original"] = 1.0 completions, penalty_counts = impl._results_to_completions([result]) assert completions[0].reward == 0.0 + # Dynamic sampling still sees the reward the penalty overwrote. + assert completions[0].reward_original == 1.0 assert penalty_counts[count_key] == 1 assert sum(penalty_counts.values()) == 1 assert impl._compute_reward_penalty_metrics(penalty_counts, 1) == {metric_name: 1.0} -def test_nemo_gym_reward_penalty_metrics_compute_fractional_rate(): +def test_nemo_reward_original_penalty_metrics_compute_fractional_rate(): impl = _nemo_gym_impl(True, {"penalize_empty_final_answer": True}) metrics = impl._compute_reward_penalty_metrics( @@ -2959,7 +2962,12 @@ def _scored_record(*rewards: float) -> PromptGroupRecord: prompt=[], extra_env_info=None, metadata={}, - completions=[SimpleNamespace(reward=reward) for reward in rewards], + completions=[ + SimpleNamespace( + reward=reward, reward_original=None, env_extras=None, truncated=False + ) + for reward in rewards + ], rollout_metrics={}, ) diff --git a/tests/unit/experience/test_rollout_reassembler_actor.py b/tests/unit/experience/test_rollout_reassembler_actor.py index 8a14482c604..dcb4ebeebb1 100644 --- a/tests/unit/experience/test_rollout_reassembler_actor.py +++ b/tests/unit/experience/test_rollout_reassembler_actor.py @@ -216,6 +216,10 @@ def test_rpc_dataclass_fields_are_classified() -> None: "canonical_output_tokens", "metrics", "reward_observations", + "reward_mask_sample", + "reward_truncated", + "reward_original", + "loss_multiplier", "dropped", "drop_reason", "valid_row_count", diff --git a/tests/unit/experience/test_rollouts.py b/tests/unit/experience/test_rollouts.py index ad29d70bf0c..82a95023276 100644 --- a/tests/unit/experience/test_rollouts.py +++ b/tests/unit/experience/test_rollouts.py @@ -2960,7 +2960,14 @@ class _DuplicateStream: def __init__(self): self.values = iter( [ - _ReadyRef((0, {"name": "agent"}, {"value": "first"}, None)), + _ReadyRef( + ( + 0, + {"name": "agent"}, + {"value": "first", "full_result": {"reward": 1.0}}, + None, + ) + ), _ReadyRef((0, {"name": "agent"}, {"value": "duplicate"}, None)), ] ) diff --git a/tests/unit/experience/test_selected_forest_boundaries.py b/tests/unit/experience/test_selected_forest_boundaries.py index 85572d59078..689488f7e9e 100644 --- a/tests/unit/experience/test_selected_forest_boundaries.py +++ b/tests/unit/experience/test_selected_forest_boundaries.py @@ -1304,6 +1304,12 @@ def variable_completion(capture: Any, active: Any, **kwargs: Any) -> Any: assert observation.rollout_id == "attempt" assert observation.reward == pytest.approx(2.0 - (scored_index + 1) / 10) assert observation.log_context.agent_name == "test-agent" + # Raw voting flags ride beside the observation, one per observation. + assert result.reward_mask_sample == [False] + assert len(result.reward_truncated) == 1 + # The Gym reward is the dispatcher's input, before effort shaping. + assert result.reward_original == [1.0] + assert result.loss_multiplier == 1.0 assert result.metrics[f"finalize/effort/low/{scored_index + 1}"] == 1 for key in result.meta.sample_ids[:3]: assert forest.plane.rows["canonical", key][ diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index 6443125c1cf..038d20b21d1 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -34,7 +34,10 @@ from nemo_rl.algorithms.single_controller import ( RolloutRuntimeState, SingleControllerActor, - _BelowValidFraction, +) +from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + ZeroVarianceGroup, ) from nemo_rl.data_plane import KVBatchMeta from nemo_rl.data_plane.schema import ROLLOUT_METRICS, ROUTE_PLAN_TAG @@ -130,8 +133,15 @@ def _controller(actor: object) -> Any: min_valid_fraction_per_group=None, forest_training=False, ), - grpo=SimpleNamespace(num_prompts_per_step=1), + grpo=SimpleNamespace( + num_prompts_per_step=1, + masked_reward_policy="include", + overlong_filtering=False, + ), ) + ctrl._algo_cfg = ctrl._master_config.grpo + ctrl._is_ppo = False + ctrl._dynamic_sampling_enabled = False ctrl._trainer_version = 3 ctrl._train_steps = 3 return ctrl @@ -501,12 +511,14 @@ async def _enrich(meta: KVBatchMeta, extra_env_info: Any) -> None: ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) ) - assert outcome == _BelowValidFraction( - valid_count=1, - total_count=2, + assert outcome == BelowValidFractionGroup( replacement=None, lineage_group_id=None, + target_step=3, reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + zero_variance=True, + valid_count=1, + total_count=2, ) assert ctrl._buffer.group_ids == () assert client.ready_during_clear == [False, False] @@ -540,17 +552,21 @@ def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut( ctrl._rollout_recovery_enabled = recovery ctrl._rollout_recovery_ledger.__contains__.return_value = True ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + ctrl._buffer.promote_ready_group = MagicMock(return_value=None) + ctrl._batch_promotions = {} outcome = asyncio.run( ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) ) - assert outcome == _BelowValidFraction( - valid_count=1, - total_count=2, + assert outcome == BelowValidFractionGroup( replacement=spare, lineage_group_id="spare-group" if recovery else None, + target_step=3, reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + zero_variance=True, + valid_count=1, + total_count=2, ) ctrl._buffer.commit_finalized.assert_not_awaited() ctrl._buffer.abort.assert_called_once_with("group") @@ -571,6 +587,244 @@ def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut( assert ctrl._rollout_runtime.batch_shortfall == {} +def test_below_threshold_replacement_borrows_a_finished_group_and_repays_the_lender() -> ( + None +): + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[{"idx": 99}]) + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + ctrl._batch_promotions = {} + ctrl._buffer.promote_ready_group = MagicMock(return_value=5) + + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) + + ctrl._buffer.promote_ready_group.assert_called_once_with(to_target_step=3) + assert ctrl._batch_promotions == {3: 1} + # The spare is owed to the lender, so it is reserved for the lender's step. + assert outcome.target_step == 5 + assert outcome.lineage_group_id == "spare-group" + (_, reserved), kwargs = ctrl._rollout_manager.reserve_prompt_group.call_args + assert reserved == {"idx": 99} + assert kwargs == {"target_step": 5} + assert ctrl._rollout_runtime.batch_shortfall == {} + + +def _signal_result( + rewards: list[float], + *, + truncated: list[bool] | None = None, + mask_sample: list[bool] | None = None, + reward_original: list[float] | None = None, +) -> FinalizedGroup: + count = len(rewards) + return FinalizedGroup( + meta=KVBatchMeta( + partition_id="canonical", + task_name="train", + sample_ids=[f"group_g{i}" for i in range(count)], + fields=["input_ids"], + sequence_lengths=[3] * count, + tags=[{"weight_version": 3}] * count, + ), + group_min_wv=3, + group_max_wv=3, + staging_keys=["group_g0/call"], + reward_observations=[ + FinalizedReward(f"group_g{i}", f"group_g{i}", reward, None) + for i, reward in enumerate(rewards) + ], + reward_mask_sample=mask_sample or [False] * count, + reward_truncated=truncated or [False] * count, + reward_original=list(rewards if reward_original is None else reward_original), + valid_row_count=count, + total_row_count=count, + ) + + +def _dynamic_sampling( + ctrl: Any, + *, + spares: list[dict[str, Any]], + exhausted: bool = False, + policy: str = "include", + overlong_filtering: bool = False, + lender_step: int | None = None, +) -> None: + ctrl._dynamic_sampling_enabled = True + ctrl._algo_cfg.masked_reward_policy = policy + ctrl._algo_cfg.overlong_filtering = overlong_filtering + ctrl._algo_cfg.dynamic_sampling_max_gen_batches = 10 + ctrl._filtered_prompt_budget = 10 * ctrl._algo_cfg.num_prompts_per_step + ctrl._filtered_prompts_by_step = {} + ctrl._batch_promotions = {} + ctrl._dataloader_exhausted = exhausted + ctrl._rollout_runtime.filtered_reserve = deque(spares) + ctrl._rollout_runtime.sampler_stamps_target_steps = True + ctrl._rollout_runtime.batch_shortfall = {} + ctrl._rollout_runtime.steps_allowed_short = set() + ctrl._rollout_recovery_enabled = False + ctrl._buffer.promote_ready_group = MagicMock(return_value=lender_step) + + +def _finalize_for_step_3(ctrl: Any) -> Any: + return asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) + + +def test_zero_variance_group_is_rejected_before_commit_and_replaced() -> None: + """The spare comes from the filtered pool and is charged to the step's budget.""" + spare = {"idx": 7} + result = _signal_result([1.0, 1.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[spare]) + + outcome = _finalize_for_step_3(ctrl) + + assert outcome == ZeroVarianceGroup( + replacement=spare, + lineage_group_id=None, + target_step=3, + reward_observations=result.reward_observations, + zero_variance=True, + ) + ctrl._buffer.commit_finalized.assert_not_awaited() + ctrl._buffer.abort.assert_called_once_with("group") + assert not ctrl._rollout_runtime.filtered_reserve + assert ctrl._filtered_prompts_by_step == {3: 1} + assert ctrl._rollout_runtime.batch_shortfall == {} + # Not a fault-tolerance replacement, so that pool is untouched. + assert not ctrl._rollout_runtime.replacement_reserve + assert ctrl._available_finalizers.qsize() == 1 + + +def test_zero_variance_group_borrows_a_finished_group_and_repays_the_lender() -> None: + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_signal_result([0.0, 0.0]))) + ) + _dynamic_sampling(ctrl, spares=[{"idx": 7}], lender_step=5) + ctrl._rollout_recovery_enabled = True + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + + outcome = _finalize_for_step_3(ctrl) + + ctrl._buffer.promote_ready_group.assert_called_once_with(to_target_step=3) + assert ctrl._batch_promotions == {3: 1} + # The spare is owed to the lender, so it is reserved for the lender's step, + # in the same cut that discarded the rejected group's ledger entry. + assert outcome.target_step == 5 + assert outcome.lineage_group_id == "spare-group" + (discard_cut, discarded), _ = ctrl._rollout_recovery_ledger.discard_group.call_args + (reserve_cut, reserved), kwargs = ( + ctrl._rollout_manager.reserve_prompt_group.call_args + ) + assert discarded == "group" + assert reserved == {"idx": 7} + assert kwargs == {"target_step": 5} + assert reserve_cut is discard_cut + + +def test_zero_variance_group_closes_its_step_short_when_no_spare_is_left() -> None: + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_signal_result([1.0, 1.0]))) + ) + _dynamic_sampling(ctrl, spares=[], exhausted=True) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) + assert outcome.replacement is None + assert ctrl._rollout_runtime.batch_shortfall == {3: 1} + assert 3 in ctrl._rollout_runtime.steps_allowed_short + ctrl._buffer.promote_ready_group.assert_not_called() + ctrl._buffer.commit_finalized.assert_not_awaited() + + +def test_group_with_reward_spread_commits_and_leaves_the_pool_alone() -> None: + result = _signal_result([1.0, 0.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}]) + + assert _finalize_for_step_3(ctrl) is result + + ctrl._buffer.commit_finalized.assert_awaited_once() + assert len(ctrl._rollout_runtime.filtered_reserve) == 1 + assert ctrl._filtered_prompts_by_step == {} + + +def test_zero_variance_group_commits_when_dynamic_sampling_is_off() -> None: + result = _signal_result([1.0, 1.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + + assert _finalize_for_step_3(ctrl) is result + + ctrl._buffer.commit_finalized.assert_awaited_once() + + +@pytest.mark.parametrize( + ("policy", "rejected"), + [("exclude", True), ("include", False)], +) +def test_only_the_voting_rows_decide_the_filter(policy: str, rejected: bool) -> None: + """The truncated 0.0 is the only spread; under exclude it does not vote.""" + result = _signal_result([0.0, 1.0, 1.0, 1.0], truncated=[True, False, False, False]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}], policy=policy, overlong_filtering=True) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) is rejected + if not rejected: + assert outcome is result + + +@pytest.mark.parametrize( + ("trained", "gym", "rejected"), + [ + # Effort shaping or a penalty made the trained rewards differ; the Gym + # reward did not, so the group carries no task signal. + ([1.9, 1.0], [1.0, 1.0], True), + ([0.0, 1.0], [1.0, 1.0], True), + # Only the trained rewards are uniform; the Gym rewards still differ. + ([0.0, 0.0], [0.0, 1.0], False), + ], +) +def test_the_filter_reads_the_original_reward_not_the_trained_reward( + trained: list[float], gym: list[float], rejected: bool +) -> None: + result = _signal_result(trained, reward_original=gym) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}]) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) is rejected + + +def test_below_valid_fraction_is_decided_before_zero_variance() -> None: + """A group that fails both takes a fault-tolerance spare, not a filtered one.""" + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[{"idx": 1}]) + _dynamic_sampling(ctrl, spares=[{"idx": 2}]) + ctrl._rollout_runtime.replacement_reserve = deque([{"idx": 1}]) + ctrl._rollout_runtime.sampler_stamps_target_steps = True + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, BelowValidFractionGroup) + assert outcome.replacement == {"idx": 1} + assert len(ctrl._rollout_runtime.filtered_reserve) == 1 + assert ctrl._filtered_prompts_by_step == {} + + def test_actor_rpc_failure_is_fatal_and_does_not_retry_or_requeue_actor() -> None: finalize = _RemoteFinalize(error=RuntimeError("actor died after submission")) actor = SimpleNamespace(finalize=finalize) diff --git a/tests/unit/single_controller/test_ppo_setup.py b/tests/unit/single_controller/test_ppo_setup.py index dd496f18025..770292fafa9 100644 --- a/tests/unit/single_controller/test_ppo_setup.py +++ b/tests/unit/single_controller/test_ppo_setup.py @@ -841,14 +841,11 @@ def test_accepts_shrink_because_filtering_replaces_regardless(self): _dynamic_sampling_config(on_dropped_prompt="shrink") ) - def test_rejects_token_capture(self): - # The capture path cannot borrow a finished group from a later step, so every - # drop would stall its step for a full fresh rollout. + def test_accepts_token_capture(self): mc = _dynamic_sampling_config() mc.token_capture.enabled = True - with pytest.raises(ValueError, match="token_capture.enabled"): - validate_single_controller_config(mc) + validate_single_controller_config(mc) def test_rejects_a_zero_generation_batch_allowance(self): mc = _dynamic_sampling_config(max_gen_batches=0) diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index 4dd53149f78..0294779977c 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -46,7 +46,6 @@ from nemo_rl.algorithms.single_controller import ( RolloutRuntimeState, SingleControllerActor, - _BelowValidFraction, ) from nemo_rl.algorithms.single_controller_utils.config import ( AsyncRLConfig, @@ -54,6 +53,10 @@ RolloutRecoveryConfig, ) from nemo_rl.algorithms.single_controller_utils.setup import SingleControllerActorArgs +from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + ZeroVarianceGroup, +) from nemo_rl.data.interfaces import DatumSpec from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.experience.rollout_manager import RolloutManager, RolloutOutcome @@ -122,6 +125,8 @@ def _init_pump_ledgers(ctrl: Any) -> None: # Dynamic sampling off: the filter never fires, so no test here draws on its # budget unless it opts in. ctrl._dynamic_sampling_enabled = False + ctrl._is_ppo = False + ctrl._filtered_spares_claimed = 0 ctrl._filtered_prompt_budget = 0 ctrl._rollout_runtime.filtered_reserve = deque() ctrl._filtered_prompts_by_step = {} @@ -1515,12 +1520,14 @@ def test_below_valid_fraction_outcome_substitutes_or_closes_short( ) -> None: """The pump acts on the finalize cut's decision instead of re-deciding it.""" spare = {"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1} - rejected = _BelowValidFraction( - valid_count=3, - total_count=4, + rejected = BelowValidFractionGroup( replacement=spare if has_spare else None, lineage_group_id="spare-group" if has_spare else None, + target_step=0, reward_observations=[], + zero_variance=True, + valid_count=3, + total_count=4, ) committed = FinalizedGroup( meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] @@ -1595,6 +1602,238 @@ async def run() -> None: assert "(3/4 < 0.8)" in capsys.readouterr().out +@pytest.mark.parametrize("has_spare", [False, True]) +def test_zero_variance_outcome_substitutes_from_the_filtered_pool_or_closes_short( + has_spare: bool, capsys: pytest.CaptureFixture[str] +) -> None: + """A filtered group draws only the dynamic-sampling budget, never a failure retry.""" + spare = {"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1} + # Step 1 lent the finished group that filled step 0, so the spare repays step 1. + rejected = ZeroVarianceGroup( + replacement=spare if has_spare else None, + lineage_group_id="spare-group" if has_spare else None, + target_step=1, + reward_observations=[], + zero_variance=True, + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=None, forest_training=False + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + request = SimpleNamespace(group_id="group") + ctrl._rollout_manager = SimpleNamespace( + generate_for_finalization=AsyncMock(return_value=(request, {})), + record_finalizer_dropped_prompt=MagicMock(), + record_group_rewards=MagicMock(), + stats=SimpleNamespace(committed=0, zero_variance_groups=0), + ) + _init_pump_ledgers(ctrl) + ctrl._finalizer_actors = [object()] + ctrl._finalize_with_actor = AsyncMock(side_effect=[rejected, committed]) + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + await ctrl._rollout_pump() + + # The spare is generated for the step the finalize cut reserved it for, and + # it does not spend the fault-tolerance budget. + assert [call.kwargs for call in ctrl._finalize_with_actor.await_args_list] == [ + {"target_step": 0, "replacements_used": 0} + ] + ([{"target_step": 1, "replacements_used": 0}] if has_spare else []) + assert ctrl._rollout_manager.stats.zero_variance_groups == 1 + assert ctrl._rollout_manager.stats.committed == int(has_spare) + assert ctrl._buffer_capacity._value == int(not has_spare) + assert ctrl._inflight_rollouts == 0 + # A filtered group is a healthy rollout, not an infrastructure drop. + ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_not_called() + generate = ctrl._rollout_manager.generate_for_finalization + if has_spare: + assert generate.await_count == 2 + assert generate.await_args.args == (spare,) + assert generate.await_args.kwargs["lineage_group_id"] == "spare-group" + # Counted against the step that was repaid. + assert ctrl._batch_replacements == {1: 1} + tallies = ctrl._rollout_manager.record_group_rewards.call_args_list + assert [call.kwargs["replacement_index"] for call in tallies] == [0, 1] + else: + assert generate.await_count == 1 + + asyncio.run(run()) + assert "its voting rewards carry no signal" in capsys.readouterr().out + + +def test_a_group_rejected_for_both_reasons_tallies_each_attempt_once( + capsys: pytest.CaptureFixture[str], +) -> None: + """Fault-tolerance and filtered replacements share one tally index sequence.""" + below_valid = BelowValidFractionGroup( + replacement={"message_log": [[{"role": "user", "content": "a"}]], "idx": 1}, + lineage_group_id=None, + target_step=0, + reward_observations=[], + zero_variance=False, + valid_count=3, + total_count=4, + ) + zero_variance = ZeroVarianceGroup( + replacement={"message_log": [[{"role": "user", "content": "b"}]], "idx": 2}, + lineage_group_id=None, + target_step=0, + reward_observations=[], + zero_variance=True, + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=0.8, forest_training=False + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + request = SimpleNamespace(group_id="group") + ctrl._rollout_manager = SimpleNamespace( + generate_for_finalization=AsyncMock(return_value=(request, {})), + record_finalizer_dropped_prompt=MagicMock(), + record_group_rewards=MagicMock(), + stats=SimpleNamespace(committed=0, zero_variance_groups=0), + ) + _init_pump_ledgers(ctrl) + ctrl._finalizer_actors = [object()] + ctrl._finalize_with_actor = AsyncMock( + side_effect=[below_valid, zero_variance, committed] + ) + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + await ctrl._rollout_pump() + + # Only the fault-tolerance replacement spends that budget. + assert [ + call.kwargs["replacements_used"] + for call in ctrl._finalize_with_actor.await_args_list + ] == [0, 1, 1] + tallies = ctrl._rollout_manager.record_group_rewards.call_args_list + assert [call.kwargs["replacement_index"] for call in tallies] == [0, 1, 2] + assert ctrl._rollout_manager.stats.zero_variance_groups == 1 + assert ctrl._rollout_manager.stats.committed == 1 + + asyncio.run(run()) + capsys.readouterr() + + +def _claiming_controller(*, spares: int, exhausted: bool = False) -> Any: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._dynamic_sampling_enabled = True + ctrl._rollout_runtime = RolloutRuntimeState() + ctrl._rollout_runtime.filtered_reserve = deque({"idx": i} for i in range(spares)) + ctrl._filtered_spares_claimed = 0 + ctrl._dataloader_exhausted = exhausted + return ctrl + + +def test_claiming_a_filtered_spare_waits_until_one_is_unclaimed() -> None: + """Two finalizing groups cannot both count on the pool's only spare.""" + + async def run() -> None: + ctrl = _claiming_controller(spares=1) + + async def second_claim() -> int: + async with ctrl._claim_filtered_spare(): + return ctrl._filtered_spares_claimed + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 1 + waiting = asyncio.create_task(second_claim()) + await asyncio.sleep(0.25) + assert not waiting.done() + # The pump refilled the pool, so a second spare can be promised. + ctrl._rollout_runtime.filtered_reserve.append({"idx": 9}) + assert await asyncio.wait_for(waiting, timeout=5) == 2 + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_a_claim_is_released_when_finalization_fails() -> None: + async def run() -> None: + ctrl = _claiming_controller(spares=1) + + with pytest.raises(RuntimeError, match="boom"): + async with ctrl._claim_filtered_spare(): + raise RuntimeError("boom") + + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_no_claim_is_made_once_the_dataloader_is_exhausted() -> None: + """Nothing will refill an empty pool, so waiting for it would hang the group.""" + + async def run() -> None: + ctrl = _claiming_controller(spares=0, exhausted=True) + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_claiming_does_nothing_without_dynamic_sampling() -> None: + async def run() -> None: + ctrl = _claiming_controller(spares=0) + ctrl._dynamic_sampling_enabled = False + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + def test_actor_path_releases_generation_permit_before_finalization() -> None: class _SplitRolloutManager: def __init__(self) -> None: @@ -1634,10 +1873,13 @@ async def _delayed_finalize( del request, rollout_metrics, target_step finalizers_started += 1 await release_finalizers.wait() - return SimpleNamespace( + return FinalizedGroup( + meta=None, + group_min_wv=0, + group_max_wv=0, + staging_keys=[], valid_row_count=1, total_row_count=1, - reward_observations=[], ) controller_cls = SingleControllerActor.__ray_metadata__.modified_class @@ -1807,10 +2049,13 @@ async def _finalize( manager.recovery_ledger.discard_group(cut, request.group_id) if not committed: return None - return SimpleNamespace( + return FinalizedGroup( + meta=None, + group_min_wv=0, + group_max_wv=0, + staging_keys=[], valid_row_count=1, total_row_count=1, - reward_observations=[], ) ctrl._finalize_with_actor = _finalize diff --git a/tests/unit/single_controller/test_step_reward_metrics.py b/tests/unit/single_controller/test_step_reward_metrics.py index d3d7337abcb..25b7c441bf7 100644 --- a/tests/unit/single_controller/test_step_reward_metrics.py +++ b/tests/unit/single_controller/test_step_reward_metrics.py @@ -19,7 +19,12 @@ import pytest from nemo_rl.data_plane.schema import MASK_SAMPLE -from nemo_rl.experience.dynamic_sampling import rewards_carry_no_signal +from nemo_rl.experience.dynamic_sampling import ( + rewards_carry_no_signal, + voting_rewards, + voting_rewards_from_finalized, + voting_rewards_from_record, +) from nemo_rl.experience.rollout_manager import RolloutManager @@ -49,9 +54,10 @@ def test_rewards_carry_no_signal(rewards, expected): assert rewards_carry_no_signal(rewards) is expected -def _completion(reward, *, truncated=False, mask_sample=False): +def _completion(reward, *, truncated=False, mask_sample=False, reward_original=None): return SimpleNamespace( reward=reward, + reward_original=reward_original, truncated=truncated, env_extras={MASK_SAMPLE: True} if mask_sample else None, ) @@ -102,6 +108,70 @@ def test_a_zero_weight_group_is_trivial_under_either_policy(): assert _manager(policy)._is_zero_variance_group(record) is True +@pytest.mark.parametrize( + ("policy", "overlong_filtering", "expected"), + [ + ("include", False, [0.0, 1.0, 0.5, 0.25]), + ("include", True, [0.0, 1.0, 0.5, 0.25]), + ("exclude", False, [0.0, 0.5, 0.25]), + ("exclude", True, [0.5, 0.25]), + (None, True, [0.0, 1.0, 0.5, 0.25]), + ], +) +def test_the_record_and_finalized_adapters_pick_the_same_voters( + policy, overlong_filtering, expected +): + """The capture path has a finalized group, not a record; both must pick the same voters.""" + completions = [ + _completion(0.0, truncated=True), + _completion(1.0, mask_sample=True), + _completion(0.5), + _completion(0.25), + ] + + from_flags = voting_rewards( + [c.reward for c in completions], + mask_sample=[c.env_extras is not None for c in completions], + truncated=[c.truncated for c in completions], + loss_multiplier=1.0, + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + + assert from_flags == expected + assert from_flags == voting_rewards_from_record( + _record(completions), + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + finalized = SimpleNamespace( + reward_original=[c.reward for c in completions], + reward_mask_sample=[c.env_extras is not None for c in completions], + reward_truncated=[c.truncated for c in completions], + loss_multiplier=1.0, + ) + assert from_flags == voting_rewards_from_finalized( + finalized, + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + + +def test_voting_rewards_read_the_original_reward_when_a_completion_has_one(): + """Effort shaping and penalties rewrite ``reward``; voting reads the Gym value.""" + completions = [ + _completion(1.9, reward_original=1.0), + _completion(0.0, reward_original=1.0), + _completion(0.5), + ] + + assert voting_rewards_from_record( + _record(completions), + masked_reward_policy="include", + overlong_filtering=False, + ) == [1.0, 1.0, 0.5] + + def test_unfiltered_reward_counts_kept_and_dropped_groups(): manager = _manager() # Two original groups: one mixed (kept), one all-zero (dropped by dynamic sampling). diff --git a/tests/unit/test_effort_shaping.py b/tests/unit/test_effort_shaping.py index b751c1001f5..607438da719 100644 --- a/tests/unit/test_effort_shaping.py +++ b/tests/unit/test_effort_shaping.py @@ -355,6 +355,8 @@ def test_run_rollouts_shapes_completion_reward_and_emits_low_metrics(): # length_reward = min(1, 1.0 * (1 - 100/1000)) = 0.9 -> 1.0 + 1.0 * 0.9 = 1.9 assert completions[0].reward == pytest.approx(1.9) + # Dynamic sampling filters on the reward from before shaping. + assert completions[0].reward_original == pytest.approx(1.0) # Downstream reward metrics are computed from the shaped Completion. assert metrics["total_reward/mean"] == pytest.approx(1.9) assert metrics["mean_length_reward_low"] == pytest.approx(0.9) From 0a1324cdf689e13242fffc359fbcd0ece02b3254 Mon Sep 17 00:00:00 2001 From: ruit Date: Fri, 13 Nov 2026 19:43:14 -0800 Subject: [PATCH 5/8] chore(sc): apply review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docstrings/descriptions: - single_controller.py: _claim_filtered_spare docstring — dataloader exhaustion stops waiting for a spare, it does not stop claiming an available one - dapo capture recipe + nightly.txt: zero-variance groups are rejected before they become selectable (the group is finalized, then its replay commit is blocked), not 'before finalization' - test_rollout_manager.py: restore the nemo_gym_* reward-penalty test names Dedup the capture zero-variance classification (own it in RolloutManager): - move the finalized-group classification to RolloutManager._is_zero_variance_finalized, beside _is_zero_variance_group, so the manager that owns the voting settings owns both paths; delete the SC _voting_rewards helper - classify once: the finalize cut sets the verdict on FinalizedGroup.zero_variance and the dispatch loop reads it instead of reclassifying; the ordinary path reuses one voting/zero-variance result for its filter check, log and record Coverage: - test_rollout_pump: a pump regression test that drives a real finalize so a zero-variance group claims and pops its filtered spare (not a mock) - test_selected_forest_boundaries: finalizer carries non-default voting fields (mixed mask_sample/truncated, non-unit loss weight, pre-shaping rewards) Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: ruit --- ...c-gym-single-controller-token_capture.yaml | 4 +- nemo_rl/algorithms/single_controller.py | 36 ++--- nemo_rl/experience/rollout_manager.py | 21 ++- nemo_rl/experience/rollout_reassembler.py | 6 + tests/test_suites/nightly.txt | 2 +- tests/unit/experience/test_rollout_manager.py | 4 +- .../test_selected_forest_boundaries.py | 49 +++++++ .../test_finalizer_lifecycle.py | 19 +++ .../single_controller/test_rollout_pump.py | 137 ++++++++++++++++++ 9 files changed, 241 insertions(+), 37 deletions(-) diff --git a/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml b/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml index 338c090d481..ed1c007043b 100644 --- a/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml +++ b/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml @@ -1,6 +1,6 @@ # DAPO-style sibling of the SingleController token-capture Gym nightly: dynamic -# sampling drops a group whose rollouts all score the same before it is -# finalized and replaces it from the filtering pool, and the reward is shaped +# sampling drops a group whose rollouts all score the same before it becomes +# selectable and replaces it from the filtering pool, and the reward is shaped # and scaled as in the DAPO recipes. This is the nightly that exercises that # rejection with real captured rows, including the cleanup of the staged rows and # the spare's ledger reservation. diff --git a/nemo_rl/algorithms/single_controller.py b/nemo_rl/algorithms/single_controller.py index 8982e8c937f..416c480e088 100644 --- a/nemo_rl/algorithms/single_controller.py +++ b/nemo_rl/algorithms/single_controller.py @@ -182,10 +182,6 @@ EnvironmentCheckpoint, ) from nemo_rl.environments.nemo_gym import should_use_nemo_gym -from nemo_rl.experience.dynamic_sampling import ( - rewards_carry_no_signal, - voting_rewards_from_finalized, -) from nemo_rl.experience.failures import RolloutStall from nemo_rl.experience.payload import VIOLATION_TAG_KEYS from nemo_rl.experience.reward_penalties import ( @@ -1360,18 +1356,6 @@ def _valid_fraction_counts( return valid_count, total_count return None - def _voting_rewards(self, finalized: "FinalizedGroup") -> list[float]: - """The group's rewards that vote in its baseline under the masking policy.""" - return voting_rewards_from_finalized( - finalized, - masked_reward_policy=( - None - if self._is_ppo - else cast(GRPOConfig, self._algo_cfg).masked_reward_policy - ), - overlong_filtering=self._algo_cfg.overlong_filtering, - ) - @contextlib.asynccontextmanager async def _claim_filtered_spare(self) -> AsyncIterator[None]: """Promise a finalizing group a filtered spare, in case it is rejected. @@ -1381,8 +1365,8 @@ async def _claim_filtered_spare(self) -> AsyncIterator[None]: holds a spare no other finalizing group has claimed. A claim only counts against the pool; the spare is popped inside the cut, and only on rejection. - No claim is made once the dataloader is exhausted: nothing will refill the - pool, so a rejected group takes whatever is left or closes its step short. + Once the dataloader is exhausted, no group waits for another spare: + rejection takes whatever remains or closes its step short. """ if not self._dynamic_sampling_enabled: yield @@ -1481,9 +1465,13 @@ async def _finalize_with_actor( else: actor_reusable = True - zero_variance = not finalized.dropped and rewards_carry_no_signal( - self._voting_rewards(finalized) + zero_variance = ( + not finalized.dropped + and self._rollout_manager._is_zero_variance_finalized(finalized) ) + # The committed FinalizedGroup carries the verdict back to the + # dispatch loop so it is classified once, not recomputed there. + finalized.zero_variance = zero_variance if finalized.dropped: try: await self._cleanup_known_finalization_request_unlocked( @@ -1888,13 +1876,7 @@ async def _dispatch_one_prompt( target_step, [row.reward for row in finalized.reward_observations], replacement_index=replacements + filtered_replacements, - zero_variance=( - finalized.zero_variance - if isinstance(finalized, RejectedGroup) - else rewards_carry_no_signal( - self._voting_rewards(finalized) - ) - ), + zero_variance=finalized.zero_variance, ) if not isinstance(finalized, RejectedGroup): self._rollout_manager.stats.committed += 1 diff --git a/nemo_rl/experience/rollout_manager.py b/nemo_rl/experience/rollout_manager.py index 2e4165f6ba2..1b390e27e09 100644 --- a/nemo_rl/experience/rollout_manager.py +++ b/nemo_rl/experience/rollout_manager.py @@ -54,6 +54,7 @@ mask_sample_flagged, record_group_rewards, rewards_carry_no_signal, + voting_rewards_from_finalized, voting_rewards_from_record, ) from nemo_rl.experience.failures import ( @@ -123,6 +124,7 @@ if TYPE_CHECKING: from nemo_rl.algorithms.single_controller_utils.config import RolloutRecoveryConfig + from nemo_rl.experience.rollout_reassembler import FinalizedGroup from nemo_rl.experience.rollout_reassembler_actor import ReassemblyRequest @@ -2515,6 +2517,16 @@ def _is_zero_variance_group(self, record: PromptGroupRecord) -> bool: """Whether this group's voters leave every row zero advantage.""" return rewards_carry_no_signal(self._voting_rewards(record)) + def _is_zero_variance_finalized(self, finalized: "FinalizedGroup") -> bool: + """Whether a finalized capture group's voters leave every row zero advantage.""" + return rewards_carry_no_signal( + voting_rewards_from_finalized( + finalized, + masked_reward_policy=self._masked_reward_policy, + overlong_filtering=self._overlong_filtering, + ) + ) + def record_group_rewards( self, target_step: Optional[int], @@ -2670,16 +2682,15 @@ async def generate_and_push( finally: if inflight_registry is not None: inflight_registry.pop(tq_group_id, None) - if self._filter_zero_variance_groups and self._is_zero_variance_group( - record - ): + voting = self._voting_rewards(record) + is_zero_variance = rewards_carry_no_signal(voting) + if self._filter_zero_variance_groups and is_zero_variance: # Filtering happens before commit, so removing the reservation writes no rows. await self._tq_buffer.remove_group(tq_group_id) # A filtered group is a healthy rollout and breaks a consecutive infrastructure-failure run. self._consecutive_infra_drops = 0 self._stats.zero_variance_groups += 1 rewards = [c.reward for c in record.completions] - voting = self._voting_rewards(record) self.record_group_rewards( target_step, rewards, @@ -2709,7 +2720,7 @@ async def generate_and_push( target_step, [c.reward for c in record.completions], replacement_index=replacement_index, - zero_variance=self._is_zero_variance_group(record), + zero_variance=is_zero_variance, ) except Exception as error: # A failed rollout must not leave an unready slot that can block an diff --git a/nemo_rl/experience/rollout_reassembler.py b/nemo_rl/experience/rollout_reassembler.py index 1d9c7b66e97..bd3c2e906fa 100644 --- a/nemo_rl/experience/rollout_reassembler.py +++ b/nemo_rl/experience/rollout_reassembler.py @@ -223,6 +223,12 @@ class FinalizedGroup: # Each trained call's (model call ID, weight version it began on, tokens it # generated), across the group's verified rollouts; empty when dropped. call_versions: list[tuple[str, int, int]] = field(default_factory=list) + # The controller-set zero-variance verdict. The reassembler ships the raw + # reward flags above; the controller classifies the group once (via + # RolloutManager, which owns the voting settings) and fills this in before + # returning the committed group, so the dispatch loop reads it instead of + # reclassifying. + zero_variance: bool = False def _capture_token_metrics( diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 2d6e9c357dc..8fa401699a0 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -153,7 +153,7 @@ tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-a # flight, the chained-prefix cache under load) and gates on finalizer health. tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh # The DAPO variant of the capture nightly above: dynamic sampling, reward shaping and -# reward scaling on, so zero-variance groups are rejected before they are finalized. +# reward scaling on, so zero-variance groups are rejected before they become selectable. tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-noncolocated-async-gym-single-controller-token_capture-r3-defer.sh diff --git a/tests/unit/experience/test_rollout_manager.py b/tests/unit/experience/test_rollout_manager.py index 807c105fa0a..2544ddf412a 100644 --- a/tests/unit/experience/test_rollout_manager.py +++ b/tests/unit/experience/test_rollout_manager.py @@ -1458,7 +1458,7 @@ def _reward_penalty_result(output, assistant_overrides=None, assistant_tokens=No ), ], ) -def test_nemo_reward_original_penalties_match_legacy_rewards_counts_and_metrics( +def test_nemo_gym_reward_penalties_match_legacy_rewards_counts_and_metrics( reward_penalty_config, output, assistant_overrides, @@ -1480,7 +1480,7 @@ def test_nemo_reward_original_penalties_match_legacy_rewards_counts_and_metrics( assert impl._compute_reward_penalty_metrics(penalty_counts, 1) == {metric_name: 1.0} -def test_nemo_reward_original_penalty_metrics_compute_fractional_rate(): +def test_nemo_gym_reward_penalty_metrics_compute_fractional_rate(): impl = _nemo_gym_impl(True, {"penalize_empty_final_answer": True}) metrics = impl._compute_reward_penalty_metrics( diff --git a/tests/unit/experience/test_selected_forest_boundaries.py b/tests/unit/experience/test_selected_forest_boundaries.py index 689488f7e9e..8d03ea45c94 100644 --- a/tests/unit/experience/test_selected_forest_boundaries.py +++ b/tests/unit/experience/test_selected_forest_boundaries.py @@ -1323,6 +1323,55 @@ def variable_completion(capture: Any, active: Any, **kwargs: Any) -> Any: ) +def test_ordinary_finalize_carries_nondefault_voting_fields( + forest: ForestCapture, monkeypatch: pytest.MonkeyPatch +) -> None: + """The finalizer ships the raw per-row voting fields the controller classifies on. + + An ordinary (non-Forest) group with one masked and one truncated row and a + non-unit dataset loss weight must return those exact flags, the pre-shaping + Gym rewards, and the loss_multiplier -- all away from their defaults -- so the + controller's zero-variance filter reads real signal, not placeholders. + """ + # Two independent single-call rollouts, each its own ordinary reward row. + records_a, _ = forest.capture([None], owner="attempt") + records_b, _ = forest.capture([None], owner="healthy") + + # Ordinary truncation is "filled max_seq_len", which the gym tokens do not + # reach; force a mixed pattern so the second row counts as truncated. + monkeypatch.setattr( + forest.finalizer, + "_truncated_flags", + lambda rows, logical: [False, True], + ) + + result = forest.finalizer.finalize_group( + "group", + ["attempt", "healthy"], + [ + receipt(records_a, terminal=records_a[-1]), + receipt(records_b, terminal=records_b[-1]), + ], + [0.25, 0.75], + mask_sample=[True, False], + fallback_weight_version=7, + prompt_idx=99, + loss_multiplier=0.5, + canonical_sample_ids=["group_g0", "group_g1"], + ) + + assert result.valid_row_count == 2 + assert len(result.reward_observations) == 2 + # The raw voting flags ride one-per-valid-row, in rollout order, not their + # defaults, so the controller classifies the real group. + assert result.reward_mask_sample == [True, False] + assert result.reward_truncated == [False, True] + # The Gym rewards, before any effort shaping or penalties. + assert result.reward_original == [0.25, 0.75] + # The dataset-level weight is carried through unchanged (non-unit). + assert result.loss_multiplier == 0.5 + + def test_publication_identity_drift_preserves_staging(forest, monkeypatch): records, _ = forest.capture([None, 0, None]) original = forest.finalizer._finalize_logical_rows diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index 038d20b21d1..b524b1e9437 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -41,6 +41,10 @@ ) from nemo_rl.data_plane import KVBatchMeta from nemo_rl.data_plane.schema import ROLLOUT_METRICS, ROUTE_PLAN_TAG +from nemo_rl.experience.dynamic_sampling import ( + rewards_carry_no_signal, + voting_rewards_from_finalized, +) from nemo_rl.experience.reward_penalties import FinalizedReward from nemo_rl.experience.rollout_reassembler import FinalizedGroup from nemo_rl.experience.rollout_reassembler_actor import ReassemblyRequest @@ -121,6 +125,21 @@ def _controller(actor: object) -> Any: ctrl._rollout_recovery_ledger = MagicMock() ctrl._rollout_recovery_ledger.__contains__.return_value = False ctrl._rollout_manager = MagicMock() + # The controller now asks the rollout manager (which owns the voting + # settings) to classify a finalized group. Mirror that classification off + # the controller's own config so the real-finalizer outcomes below stay + # driven by the group's rewards, not a truthy MagicMock. + ctrl._rollout_manager._is_zero_variance_finalized.side_effect = ( + lambda finalized: rewards_carry_no_signal( + voting_rewards_from_finalized( + finalized, + masked_reward_policy=( + None if ctrl._is_ppo else ctrl._algo_cfg.masked_reward_policy + ), + overlong_filtering=ctrl._algo_cfg.overlong_filtering, + ) + ) + ) ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._buffer = MagicMock() ctrl._buffer.commit_finalized = AsyncMock() diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index 0294779977c..ffbf5b409a0 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -87,6 +87,11 @@ from tests.unit.single_controller._rollout_checkpoints import ( attach_rollout_checkpointing, ) +from tests.unit.single_controller.test_finalizer_lifecycle import ( + _DataPlaneClient, + _request, + _signal_result, +) def _failure_cfg( @@ -1834,6 +1839,138 @@ async def run() -> None: asyncio.run(run()) +def test_pump_claims_and_pops_a_filtered_spare_for_a_real_zero_variance_rejection() -> ( + None +): + """The pump drives a real finalize that classifies and pops the filtered spare. + + This covers the ``_claim_filtered_spare`` wrapper around the real + ``_finalize_with_actor``: a finalized group whose voting rewards carry no signal + is classified (not mocked) inside the finalize cut, rejected, its spare popped + from the filtered pool, and the step refilled by the committed replacement. + """ + # The first group is uniform (zero variance -> rejected); its filtered spare is + # the second group, which has reward spread and commits. + results = iter([_signal_result([1.0, 1.0]), _signal_result([1.0, 0.0])]) + + class _ClassifyingRolloutManager: + """Owns the voting settings and classifies finalized capture groups.""" + + def __init__(self) -> None: + self._masked_reward_policy = "include" + self._overlong_filtering = False + self.stats = SimpleNamespace(committed=0, zero_variance_groups=0) + self.record_group_rewards = MagicMock() + self.record_finalizer_dropped_prompt = MagicMock() + self.record_canonical_publication = MagicMock() + self.generated: list[Any] = [] + + def _is_zero_variance_finalized(self, finalized: FinalizedGroup) -> bool: + return RolloutManager._is_zero_variance_finalized(self, finalized) + + async def generate_for_finalization( + self, prompt: Any, *, target_step: int | None = None, **kwargs: Any + ) -> Any: + del target_step, kwargs + self.generated.append(prompt) + return _request(), {} + + manager = _ClassifyingRolloutManager() + + class _ResultActor: + """Finalizer actor returning the next pre-built result each call.""" + + def __init__(self) -> None: + self.finalize = SimpleNamespace(remote=self._remote) + + async def _remote(self, request: Any) -> FinalizedGroup: + del request + return next(results) + + actor = _ResultActor() + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1, num_prompts_per_step=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=None, + forest_training=False, + staging_partition="staging", + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + ctrl._is_ppo = False + ctrl._partition_id = "canonical" + ctrl._dp_client = _DataPlaneClient() + ctrl._rollout_manager = manager + _init_pump_ledgers(ctrl) + + # The real finalize path: a fake actor pool returning the pre-built + # group, a real checkpoint barrier, and a mocked buffer/ledger so the + # commit of the replacement does not need a live data plane. + ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() + ctrl._available_finalizers = asyncio.Queue() + ctrl._available_finalizers.put_nowait(actor) + ctrl._active_finalizers = 0 + ctrl._finalizer_waiters = 0 + ctrl._finalizer_unknown_outcomes = 0 + ctrl._rollout_recovery_ledger = MagicMock() + ctrl._rollout_recovery_ledger.__contains__.return_value = False + ctrl._rollout_recovery_enabled = False + ctrl._refit_cuts = SimpleNamespace(tokens_by_version=lambda _: {}) + + # Dynamic sampling on, with one spare in the filtered pool for the + # zero-variance rejection to consume. + ctrl._dynamic_sampling_enabled = True + ctrl._dataloader_exhausted = False + ctrl._filtered_spares_claimed = 0 + ctrl._filtered_prompt_budget = 10 + ctrl._filtered_prompts_by_step = {} + ctrl._rollout_runtime.filtered_reserve = deque( + [{"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1}] + ) + ctrl._rollout_runtime.steps_allowed_short = set() + + ctrl._buffer = _RecordingBuffer() + ctrl._buffer.commit_finalized = AsyncMock() + ctrl._buffer.abort = MagicMock() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._finalizer_actors = [actor] + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + await ctrl._rollout_pump() + + # The zero-variance group was classified and rejected, its spare popped + # from the filtered pool and charged, and the replacement committed. + assert manager.stats.zero_variance_groups == 1 + assert manager.stats.committed == 1 + assert not ctrl._rollout_runtime.filtered_reserve + assert ctrl._filtered_prompts_by_step == {0: 1} + assert ctrl._filtered_spares_claimed == 0 + # Two generations: the rejected group and its filtered replacement. + assert len(manager.generated) == 2 + # Both tallies recorded; the rejection carried the zero-variance verdict. + tallies = manager.record_group_rewards.call_args_list + assert [call.kwargs["zero_variance"] for call in tallies] == [True, False] + + asyncio.run(run()) + + def test_actor_path_releases_generation_permit_before_finalization() -> None: class _SplitRolloutManager: def __init__(self) -> None: From a1cc00b177b6f7683a95f4b108c839af12029373 Mon Sep 17 00:00:00 2001 From: ruit Date: Fri, 13 Nov 2026 23:04:55 -0800 Subject: [PATCH 6/8] fix(sc): register FinalizedGroup.zero_variance in the RPC field-inventory test The review-nits commit added a controller-set zero_variance verdict field to FinalizedGroup but did not update test_rpc_dataclass_fields_are_classified, which pins the exact field inventory, so L0 unit tests failed. Register the new field (a light bool that is safe to cross the wire). Signed-off-by: ruit --- tests/unit/experience/test_rollout_reassembler_actor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/experience/test_rollout_reassembler_actor.py b/tests/unit/experience/test_rollout_reassembler_actor.py index dcb4ebeebb1..136ef67655f 100644 --- a/tests/unit/experience/test_rollout_reassembler_actor.py +++ b/tests/unit/experience/test_rollout_reassembler_actor.py @@ -227,6 +227,8 @@ def test_rpc_dataclass_fields_are_classified() -> None: "token_metrics", # One (model call ID, weight version, tokens) tuple per trained call. "call_versions", + # Controller-set verdict: the finalized group scored all-same reward. + "zero_variance", } From ae65f4ae1c59154917470fb4505aa5f7f7cba2ce Mon Sep 17 00:00:00 2001 From: ruit Date: Fri, 13 Nov 2026 23:05:25 -0800 Subject: [PATCH 7/8] test(sc): retarget the DAPO SC capture nightly to the #4588 vLLM/Gym recipe Replace the Megatron-generation 2n8g DAPO capture nightly with one that inherits the #4588 dynamic-sampling recipe (Lightning-30B, Megatron training, vLLM generation, 4n8g) and routes the math task through NeMo-Gym (OpenMathReasoning via math_with_judge) so token capture is exercised on the faster vLLM path. MAX_STEPS=5 and the check_metrics thresholds are calibrated on a completed Gym run (unfiltered reward ~0.25, token_mult_prob_error median 1.018/max 1.029, ~1040s/step). Nightly GPU-hour budget stays under 5310 (5283). Signed-off-by: ruit --- ...c-gym-single-controller-token_capture.yaml | 31 -------- ...c-gym-single-controller-token_capture.yaml | 76 +++++++++++++++++++ ...nc-gym-single-controller-token_capture.sh} | 70 ++++++++--------- tests/test_suites/nightly.txt | 4 +- 4 files changed, 110 insertions(+), 71 deletions(-) delete mode 100644 examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml create mode 100644 examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml rename tests/test_suites/llm/{dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh => dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh} (55%) diff --git a/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml b/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml deleted file mode 100644 index ed1c007043b..00000000000 --- a/examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# DAPO-style sibling of the SingleController token-capture Gym nightly: dynamic -# sampling drops a group whose rollouts all score the same before it becomes -# selectable and replaces it from the filtering pool, and the reward is shaped -# and scaled as in the DAPO recipes. This is the nightly that exercises that -# rejection with real captured rows, including the cleanup of the staged rows and -# the spare's ledger reservation. -# Launch with: -# uv run examples/run_grpo_single_controller.py \ -# --config examples/configs/recipes/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml -defaults: ./grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.yaml - -grpo: - use_dynamic_sampling: true - # Map the 0/1 task reward to [-1, 1] before the overlong penalty is subtracted. - reward_scaling: - enabled: true - target_min: -1.0 - # Soft penalty on the last 512 tokens before the generation limit. - reward_shaping: - enabled: true - overlong_buffer_length: 512 - overlong_buffer_penalty: 1 - max_response_length: ${policy.generation.max_new_tokens} - -checkpointing: - checkpoint_dir: results/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture - -logger: - log_dir: logs/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture - wandb: - name: dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture diff --git a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml new file mode 100644 index 00000000000..d4b80df4a97 --- /dev/null +++ b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml @@ -0,0 +1,76 @@ +# DAPO SingleController token-capture Gym nightly: the capture sibling of the +# #4588 dynamic-sampling nightly it inherits. Same Lightning-30B model, Megatron +# training, vLLM generation, DAPO reward shaping/scaling, dynamic sampling and 4n8g +# topology. The only difference: the math task runs through NeMo-Gym (math_with_judge +# / OpenMathReasoning) instead of #4588's built-in math env, because token capture is +# only wired up on the Gym rollout path. Math keeps the reward variance the +# zero-variance filter needs so it does not saturate. +# Launch with: +# uv run examples/run_grpo_single_controller.py \ +# --config examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml +defaults: ./dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml +token_capture: + enabled: true +policy: + generation: + vllm_cfg: + expose_http_server: true +# Route the DAPO math task through NeMo-Gym for capture (OpenMathReasoning via +# math_with_judge, HF-downloadable so CI can ng_prepare_data it). Judge off: +# math-verify alone scores the rows, keeping the reward binary. +env: + _override_: true + math: + num_workers: 16 + math_verify_impl: dapo_math_verify + should_use_nemo_gym: true + nemo_gym: + config_paths: + - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml + - resources_servers/math_with_judge/configs/math_with_judge.yaml + math_with_judge: + resources_servers: + math_with_judge: + judge_model_server: + name: policy_model + should_use_judge: false + policy_model: + responses_api_models: + vllm_model: + uses_reasoning_parser: false +data: + _override_: true + max_input_seq_length: 2048 + shuffle: true + num_workers: 0 + use_multiple_dataloader: false + default: + dataset_name: NemoGymDataset + env_name: nemo_gym + processor: nemo_gym_data_processor + prompt_file: null + system_prompt_file: null + train: + env_name: nemo_gym + data_path: 3rdparty/Gym-workspace/Gym/data/math_with_judge/train.jsonl + processor: nemo_gym_data_processor + dataset_name: NemoGymDataset + prompt_file: null + system_prompt_file: null + split_validation_size: 0 + # SC has no validation (val_period=0), so this is never read; point it at the + # train rows to avoid depending on a second materialized file. + validation: + env_name: nemo_gym + data_path: 3rdparty/Gym-workspace/Gym/data/math_with_judge/train.jsonl + processor: nemo_gym_data_processor + dataset_name: NemoGymDataset + prompt_file: null + system_prompt_file: null + split_validation_size: 0 +checkpointing: + checkpoint_dir: results/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture +logger: + log_dir: logs/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture + wandb: + name: dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture diff --git a/tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh b/tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh similarity index 55% rename from tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh rename to tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh index 09f4ef91d79..79c7c7e3e27 100755 --- a/tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh +++ b/tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh @@ -16,25 +16,27 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) source $SCRIPT_DIR/common.env # ===== BEGIN CONFIG ===== -NUM_NODES=2 +NUM_NODES=4 GPUS_PER_NODE=8 STEPS_PER_RUN=10 -MAX_STEPS=3 +MAX_STEPS=5 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up # ~25 min startup (30B-MoE load + CUDA-graph warmup + nemo_gym servers) plus -# ~16 min/step, and dynamic sampling generates extra groups to replace the -# filtered ones. Three steps still cover a refit with requests in flight and a -# checkpoint save through the finalizer. -NUM_MINUTES=120 # Initial budget; calibrate against a completed run. +# ~17 min/step on OpenMathReasoning, and dynamic sampling generates extra groups +# to replace the filtered ones. Five steps cover a refit with requests in flight +# and a checkpoint save through the finalizer. Calibrated against job 21912385 +# (six steps in ~115 min including startup). +NUM_MINUTES=130 # Initial 4-node budget; calibrate against a completed run. # ===== END CONFIG ===== exit_if_max_steps_reached cd $PROJECT_ROOT -# Prepare nemo-gym workplace_assistant dataset (mirrors tests/functional/grpo_async_gym.sh). -DATA_DIR=$EXP_DIR/data -mkdir -p $DATA_DIR +# Prepare the NeMo-Gym math_with_judge (OpenMathReasoning) taskset. math_with_judge +# downloads from the HF repo nvidia/Nemotron-RL-math-OpenMathReasoning, so CI can +# materialize it; the recipe reads the train rows written here. Mirrors the +# workplace_assistant prep in grpo_async_gym.sh but needs no tool trimming. cd 3rdparty/Gym-workspace/Gym if [[ ! -f env.yaml ]]; then if [[ -z "${HF_TOKEN:-}" ]]; then @@ -43,19 +45,13 @@ if [[ ! -f env.yaml ]]; then fi echo "hf_token: $HF_TOKEN" >> env.yaml fi -uv run ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml]" \ - +output_dirpath=data/workplace_assistant \ +uv run ng_prepare_data "+config_paths=[resources_servers/math_with_judge/configs/math_with_judge.yaml]" \ + +output_dirpath=data/math_with_judge \ +mode=train_preparation \ +should_download=true \ +data_source=huggingface cd - -# Trim to a single tool so first prompt fits. -TRAIN_PATH=$DATA_DIR/workplace_assistant_train.jsonl -VALIDATION_PATH=$DATA_DIR/workplace_assistant_validation.jsonl -jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/train.jsonl > $TRAIN_PATH -jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/validation.jsonl > $VALIDATION_PATH - # Token capture is only wired up on the SingleController path. uv run examples/run_grpo_single_controller.py \ --config $CONFIG_PATH \ @@ -69,8 +65,6 @@ uv run examples/run_grpo_single_controller.py \ checkpointing.enabled=True \ checkpointing.checkpoint_dir=$CKPT_DIR \ checkpointing.save_period=3 \ - data.train.data_path=$TRAIN_PATH \ - data.validation.data_path=$VALIDATION_PATH \ $@ \ 2>&1 | tee $RUN_LOG @@ -80,35 +74,37 @@ uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS # (dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.sh, #4588), # plus the capture-only finalize/* health gates this recipe adds. # -# The dynamic-sampling gates: the filter has to be doing something (>=1 zero-variance -# group filtered across the run) and has to be paying for it without shrinking the step -# (skipped_step and dropped_prompt_groups stay 0 -- the batch size the feature exists to -# hold). unfiltered_reward, not reward, is the progress signal: reward here is the mean -# over the trained (kept) rows, which excludes the all-same-reward groups, so it is the -# wrong denominator to gate progress on (v1 reports the two under swapped names; see -# docs/guides/single-controller.md#dynamic-sampling). +# The dynamic-sampling gates: the filter has to be doing something (zero-variance +# groups accumulate across the run) and has to be paying for it without shrinking the +# step (skipped_step and dropped_prompt_groups stay 0 -- the batch size the feature +# exists to hold). unfiltered_reward, not reward, is the progress signal: reward here +# is the mean over the trained (kept) rows, which excludes the all-same-reward groups, +# so it is the wrong denominator to gate progress on (v1 reports the two under swapped +# names; see docs/guides/single-controller.md#dynamic-sampling). # # The capture gates: a nonzero finalize/invalid_row_rate or capture_poisoned_rollouts # means capture or recovery lost a rollout's tokens. # -# Thresholds are for this recipe's scale (3 steps, 2n8g 30B, Megatron generation): -# total_step_time matches the capture nightly (<1000) plus headroom for the extra -# replacement generation dynamic sampling drives; no max(total_step_time) gate because -# save_period=3 bundles a checkpoint write into the final step. zero_variance >=1 and -# unfiltered_reward >0 are intentionally loose pending calibration against a completed run. +# Thresholds are calibrated on job 21912385 (Lightning-30B, Gym OpenMathReasoning, +# vLLM generation, 4n8g), reading its six completed steps: unfiltered_reward 0.21-0.29, +# per-step zero-variance groups 10-27 (>=20 cumulative over five steps with margin), +# token_mult_prob_error median 1.018 / max 1.029 (looser than #4588's built-in-math +# bounds -- Gym rollouts through the OpenAI server drift slightly more), total_step_time +# ~1040s. No max(total_step_time) gate because save_period=3 bundles a checkpoint write +# into one step; median excludes it. uv run tests/check_metrics.py $JSON_METRICS \ "len(data[\"train/loss\"]) == $MAX_STEPS" \ 'all_finite(data["train/loss"])' \ 'all_finite(data["train/token_mult_prob_error"])' \ - 'median(data["train/token_mult_prob_error"]) < 1.02' \ - 'max(data["train/token_mult_prob_error"]) < 1.025' \ - 'median(data["train/unfiltered_reward"]) > 0.0' \ - 'median(data["timing/train/total_step_time"]) < 1500' \ + 'median(data["train/token_mult_prob_error"]) < 1.03' \ + 'max(data["train/token_mult_prob_error"]) < 1.05' \ + 'median(data["train/unfiltered_reward"]) > 0.1' \ 'max(data["train/finalize/invalid_row_rate"]) == 0' \ 'max(data["train/finalize/capture_poisoned_rollouts"]) == 0' \ - 'max(data["rollout/zero_variance_groups_total"]) >= 1' \ + 'max(data["rollout/zero_variance_groups_total"]) >= 20' \ 'min(data["train/skipped_step"]) == max(data["train/skipped_step"]) == 0' \ - 'min(data["train/dropped_prompt_groups"]) == max(data["train/dropped_prompt_groups"]) == 0' + 'min(data["train/dropped_prompt_groups"]) == max(data["train/dropped_prompt_groups"]) == 0' \ + 'median(data["timing/train/total_step_time"]) < 1500' # Clean up checkpoint directory after successful run to save space. rm -rf "$CKPT_DIR" diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 8fa401699a0..e6b3e6395a2 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -152,9 +152,6 @@ tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-a # nightly that exercises Megatron-inference capture (refits with requests in # flight, the chained-prefix cache under load) and gates on finalizer health. tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh -# The DAPO variant of the capture nightly above: dynamic sampling, reward shaping and -# reward scaling on, so zero-variance groups are rejected before they become selectable. -tests/test_suites/llm/dapo-nanov3-30BA3B-2n8g-megatron_generation-noncolocated-async-gym-single-controller-token_capture.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-noncolocated-async-gym-single-controller-token_capture-r3-defer.sh # Nano-v3.5 @@ -218,6 +215,7 @@ tests/test_suites/llm/grpo-qwen3-8b-base-1n8g-megatron-lora-tq_nixl.sh # Single Controller (SC) tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.sh +tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller-streaming2.sh tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller-sync.sh tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron_generation-noncolocated-single-controller-async-lag4.sh From d77a1b5879367ebcedca153a41c5333b6a202d38 Mon Sep 17 00:00:00 2001 From: ruit Date: Sat, 14 Nov 2026 02:05:02 -0800 Subject: [PATCH 8/8] chore(sc): apply review follow-ups - Fix the voting-field finalizer test: give the healthy receipt its matching capture_key and add an explicit invalid row between the two valid rows, so it no longer trips the identity check (was failing CI with assert 1 == 2) and now proves the voting lists skip the invalid row. - Observe the filtered-spare claim count during capture finalization in the pump composition test and assert it is held (1) across the cut, so removing the pump's claim wrapper now fails the test. - Delete the dead _is_zero_variance_group wrapper (no production callers after the classify-once refactor) and point its reward tests at the live voter/predicate path. - Correct the single-controller doc: a filtered capture group is rejected after its canonical rows are written, not before publication. - Shorten two over-long in-function comment blocks. Signed-off-by: ruit --- docs/guides/single-controller.md | 2 +- nemo_rl/experience/rollout_manager.py | 4 --- .../test_selected_forest_boundaries.py | 28 +++++++++++-------- .../test_finalizer_lifecycle.py | 5 +--- .../single_controller/test_rollout_pump.py | 17 +++++++++-- .../test_step_reward_metrics.py | 14 +++++----- 6 files changed, 39 insertions(+), 31 deletions(-) diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index 0ad850130fa..57e6b155c50 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -335,7 +335,7 @@ Set `grpo.use_dynamic_sampling: true` to drop prompt groups whose generations al - `async_rl.sampler.name: in_order`. A filtered group is charged to the target step the sampler stamps, and the spare pool is refilled while the pump waits to admit; no other built-in supports both, and a `custom` class cannot declare at setup that it does. - `grpo.num_generations_per_prompt >= 2`. -With `token_capture.enabled: true` the rewards are final only after the finalizer has verified the group's rows, so the filter runs there: a group whose voting rewards carry no signal is rejected before it is published, its rows are cleared, and it never becomes selectable. The replacement, the borrow and the per-step allowance work as above. Which rows vote follows `grpo.masked_reward_policy`, `grpo.overlong_filtering` and the environment's `mask_sample` flags, as on the non-capture path; a row that failed capture verification never votes. A filtered group has already paid for row assembly, so a high drop rate costs more here than without capture. `token_capture.min_valid_fraction_per_group` is checked first, and a group that fails it is replaced from the fault-tolerance pool (`async_rl.rollout_failure`), not the filtering pool. +With `token_capture.enabled: true` the rewards are final only after the finalizer has verified the group's rows, so the filter runs there: a group whose voting rewards carry no signal is rejected after its canonical rows are written but before it becomes selectable, and those rows are cleared. The replacement, the borrow and the per-step allowance work as above. Which rows vote follows `grpo.masked_reward_policy`, `grpo.overlong_filtering` and the environment's `mask_sample` flags, as on the non-capture path; a row that failed capture verification never votes. A filtered group has already paid for row assembly, so a high drop rate costs more here than without capture. `token_capture.min_valid_fraction_per_group` is checked first, and a group that fails it is replaced from the fault-tolerance pool (`async_rl.rollout_failure`), not the filtering pool. SC pre-warms its filtering pool with one dataloader batch (`grpo.num_prompts_per_step` raw prompts). Below that low-water mark it adds a whole batch, so the pool can hold more than PPS prompts. An empty pool is refilled on demand, including while admission or dispatch is blocked. This is an internal policy, independent of the fault-tolerance pool controlled by `async_rl.rollout_failure.replacement_reserve_prompts`; only `grpo.dynamic_sampling_max_gen_batches` limits the generation attempts for a training step. diff --git a/nemo_rl/experience/rollout_manager.py b/nemo_rl/experience/rollout_manager.py index 1b390e27e09..008dfea3cd6 100644 --- a/nemo_rl/experience/rollout_manager.py +++ b/nemo_rl/experience/rollout_manager.py @@ -2513,10 +2513,6 @@ def _voting_rewards(self, record: PromptGroupRecord) -> list[float]: overlong_filtering=self._overlong_filtering, ) - def _is_zero_variance_group(self, record: PromptGroupRecord) -> bool: - """Whether this group's voters leave every row zero advantage.""" - return rewards_carry_no_signal(self._voting_rewards(record)) - def _is_zero_variance_finalized(self, finalized: "FinalizedGroup") -> bool: """Whether a finalized capture group's voters leave every row zero advantage.""" return rewards_carry_no_signal( diff --git a/tests/unit/experience/test_selected_forest_boundaries.py b/tests/unit/experience/test_selected_forest_boundaries.py index 8d03ea45c94..f7032e69d50 100644 --- a/tests/unit/experience/test_selected_forest_boundaries.py +++ b/tests/unit/experience/test_selected_forest_boundaries.py @@ -1328,38 +1328,42 @@ def test_ordinary_finalize_carries_nondefault_voting_fields( ) -> None: """The finalizer ships the raw per-row voting fields the controller classifies on. - An ordinary (non-Forest) group with one masked and one truncated row and a - non-unit dataset loss weight must return those exact flags, the pre-shaping - Gym rewards, and the loss_multiplier -- all away from their defaults -- so the - controller's zero-variance filter reads real signal, not placeholders. + An ordinary (non-Forest) group with a masked valid row, an invalid row and a + truncated valid row, under a non-unit dataset loss weight, must return those exact + flags, the pre-shaping Gym rewards and the loss_multiplier for the valid rows only + -- all away from their defaults -- so the controller's zero-variance filter reads + real signal, not placeholders, and the invalid row never votes. """ # Two independent single-call rollouts, each its own ordinary reward row. records_a, _ = forest.capture([None], owner="attempt") records_b, _ = forest.capture([None], owner="healthy") - # Ordinary truncation is "filled max_seq_len", which the gym tokens do not - # reach; force a mixed pattern so the second row counts as truncated. + # Keep truncation mixed across the two valid reward rows. monkeypatch.setattr( forest.finalizer, "_truncated_flags", - lambda rows, logical: [False, True], + lambda rows, logical: [False, False, True], ) result = forest.finalizer.finalize_group( "group", - ["attempt", "healthy"], + ["attempt", "bad", "healthy"], [ receipt(records_a, terminal=records_a[-1]), - receipt(records_b, terminal=records_b[-1]), + # An invalid row between the two valid ones: the voting lists must skip it. + None, + # capture_key must match the rollout_id "healthy" or finalize rejects the row. + {**receipt(records_b, terminal=records_b[-1]), "capture_key": "healthy"}, ], - [0.25, 0.75], - mask_sample=[True, False], + [0.25, 7.0, 0.75], + mask_sample=[True, False, False], fallback_weight_version=7, prompt_idx=99, loss_multiplier=0.5, - canonical_sample_ids=["group_g0", "group_g1"], + canonical_sample_ids=["group_g0", "group_g1", "group_g2"], ) + assert result.total_row_count == 3 assert result.valid_row_count == 2 assert len(result.reward_observations) == 2 # The raw voting flags ride one-per-valid-row, in rollout order, not their diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index b524b1e9437..52a5cee2763 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -125,10 +125,7 @@ def _controller(actor: object) -> Any: ctrl._rollout_recovery_ledger = MagicMock() ctrl._rollout_recovery_ledger.__contains__.return_value = False ctrl._rollout_manager = MagicMock() - # The controller now asks the rollout manager (which owns the voting - # settings) to classify a finalized group. Mirror that classification off - # the controller's own config so the real-finalizer outcomes below stay - # driven by the group's rewards, not a truthy MagicMock. + # Classify with real reward helpers instead of a truthy MagicMock. ctrl._rollout_manager._is_zero_variance_finalized.side_effect = ( lambda finalized: rewards_carry_no_signal( voting_rewards_from_finalized( diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index ffbf5b409a0..090b0cba8f6 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -1909,9 +1909,7 @@ async def run() -> None: ctrl._rollout_manager = manager _init_pump_ledgers(ctrl) - # The real finalize path: a fake actor pool returning the pre-built - # group, a real checkpoint barrier, and a mocked buffer/ledger so the - # commit of the replacement does not need a live data plane. + # Run real controller finalization with fake actor and data-plane endpoints. ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._available_finalizers = asyncio.Queue() ctrl._available_finalizers.put_nowait(actor) @@ -1953,6 +1951,17 @@ async def run() -> None: ctrl._trainer_version = 0 ctrl._current_epoch = 0 + # The pump must claim the spare before the finalize cut pops it, so record + # the claim count held inside finalization. + claims_during_finalize: list[int] = [] + finalize = ctrl._finalize_with_actor + + async def finalize_with_claim(*args: Any, **kwargs: Any) -> Any: + claims_during_finalize.append(ctrl._filtered_spares_claimed) + return await finalize(*args, **kwargs) + + ctrl._finalize_with_actor = finalize_with_claim + await ctrl._rollout_pump() # The zero-variance group was classified and rejected, its spare popped @@ -1961,6 +1970,8 @@ async def run() -> None: assert manager.stats.committed == 1 assert not ctrl._rollout_runtime.filtered_reserve assert ctrl._filtered_prompts_by_step == {0: 1} + # The pump held the spare's claim across finalization, then released it. + assert claims_during_finalize[0] == 1 assert ctrl._filtered_spares_claimed == 0 # Two generations: the rejected group and its filtered replacement. assert len(manager.generated) == 2 diff --git a/tests/unit/single_controller/test_step_reward_metrics.py b/tests/unit/single_controller/test_step_reward_metrics.py index 25b7c441bf7..3750741aef1 100644 --- a/tests/unit/single_controller/test_step_reward_metrics.py +++ b/tests/unit/single_controller/test_step_reward_metrics.py @@ -69,9 +69,9 @@ def _record(completions, *, loss_multiplier=1.0): def test_zero_variance_group_predicate_reads_completion_rewards(): record = _record([_completion(1.0), _completion(1.0)]) - assert _manager()._is_zero_variance_group(record) is True + assert rewards_carry_no_signal(_manager()._voting_rewards(record)) is True record.completions.append(_completion(0.0)) - assert _manager()._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(_manager()._voting_rewards(record)) is False def test_only_the_voting_rows_decide_whether_a_group_is_trivial(): @@ -88,15 +88,15 @@ def test_only_the_voting_rows_decide_whether_a_group_is_trivial(): excluding = _manager("exclude", overlong_filtering=True) including = _manager("include", overlong_filtering=True) - assert excluding._is_zero_variance_group(record) is True - assert including._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(excluding._voting_rewards(record)) is True + assert rewards_carry_no_signal(including._voting_rewards(record)) is False def test_a_flagged_row_only_votes_under_include(): record = _record([_completion(0.0, mask_sample=True), _completion(1.0)]) - assert _manager("exclude")._is_zero_variance_group(record) is True - assert _manager("include")._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(_manager("exclude")._voting_rewards(record)) is True + assert rewards_carry_no_signal(_manager("include")._voting_rewards(record)) is False def test_a_zero_weight_group_is_trivial_under_either_policy(): @@ -105,7 +105,7 @@ def test_a_zero_weight_group_is_trivial_under_either_policy(): record = _record([_completion(0.0), _completion(1.0)], loss_multiplier=0.0) for policy in ("include", "exclude", None): - assert _manager(policy)._is_zero_variance_group(record) is True + assert rewards_carry_no_signal(_manager(policy)._voting_rewards(record)) is True @pytest.mark.parametrize(