diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index 779a6524480..57e6b155c50 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -327,14 +327,15 @@ Field definitions: ## Dynamic sampling -Set `grpo.use_dynamic_sampling: true` to drop prompt groups whose generations all received the same reward (zero within-group variance, so no advantage signal) and resample a substitute, as in [DAPO](dapo.md). A dropped group is always replaced, and the step it was stamped for is filled immediately from a later step's finished group where one exists, so the trainer does not wait out the resample. Setup requires: +Set `grpo.use_dynamic_sampling: true` to drop prompt groups whose generations all received the same reward (zero within-group variance, so no advantage signal) and resample a substitute, as in [DAPO](dapo.md). The filter reads the environment's reward as the environment returned it, before `env.nemo_gym.effort_levels` shaping and `reward_penalties`, on both rollout paths. A group whose rewards differ only because of that shaping or a penalty is therefore dropped, even though the trained rewards differ. A dropped group is always replaced, and the step it was stamped for is filled immediately from a later step's finished group where one exists, so the trainer does not wait out the resample. Setup requires: - `grpo.dynamic_sampling_max_gen_batches >= 1` (default `10`). A step may consume this many generation batches in total, the batch it was admitted with included, counted in prompts. Past that the run raises, as v1 does: a step that cannot be filled within its allowance means the filter is rejecting faster than the data can refill it. This is a separate budget from `async_rl.rollout_failure.max_replacement_attempts`, which covers rollout failures only; neither drop spends the other's allowance. Size it from the drop rate the run reports. At drop rate `p` a step needs about `1 / (1 - p)` prompts per committed group, so `grpo.dynamic_sampling_max_gen_batches` has to exceed `1 / (1 - p)` with room for the variance around it. Read `p` off `zero_variance_fraction`. - `async_rl.sampler.name: in_order`. A filtered group is charged to the target step the sampler stamps, and the spare pool is refilled while the pump waits to admit; no other built-in supports both, and a `custom` class cannot declare at setup that it does. - `grpo.num_generations_per_prompt >= 2`. -- `token_capture.enabled: false`. The capture path cannot borrow a finished group from a later step, so each drop would stall its step for a full fresh rollout; setup rejects the combination rather than paying that on every step. + +With `token_capture.enabled: true` the rewards are final only after the finalizer has verified the group's rows, so the filter runs there: a group whose voting rewards carry no signal is rejected after its canonical rows are written but before it becomes selectable, and those rows are cleared. The replacement, the borrow and the per-step allowance work as above. Which rows vote follows `grpo.masked_reward_policy`, `grpo.overlong_filtering` and the environment's `mask_sample` flags, as on the non-capture path; a row that failed capture verification never votes. A filtered group has already paid for row assembly, so a high drop rate costs more here than without capture. `token_capture.min_valid_fraction_per_group` is checked first, and a group that fails it is replaced from the fault-tolerance pool (`async_rl.rollout_failure`), not the filtering pool. SC pre-warms its filtering pool with one dataloader batch (`grpo.num_prompts_per_step` raw prompts). Below that low-water mark it adds a whole batch, so the pool can hold more than PPS prompts. An empty pool is refilled on demand, including while admission or dispatch is blocked. This is an internal policy, independent of the fault-tolerance pool controlled by `async_rl.rollout_failure.replacement_reserve_prompts`; only `grpo.dynamic_sampling_max_gen_batches` limits the generation attempts for a training step. diff --git a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml new file mode 100644 index 00000000000..d4b80df4a97 --- /dev/null +++ b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml @@ -0,0 +1,76 @@ +# DAPO SingleController token-capture Gym nightly: the capture sibling of the +# #4588 dynamic-sampling nightly it inherits. Same Lightning-30B model, Megatron +# training, vLLM generation, DAPO reward shaping/scaling, dynamic sampling and 4n8g +# topology. The only difference: the math task runs through NeMo-Gym (math_with_judge +# / OpenMathReasoning) instead of #4588's built-in math env, because token capture is +# only wired up on the Gym rollout path. Math keeps the reward variance the +# zero-variance filter needs so it does not saturate. +# Launch with: +# uv run examples/run_grpo_single_controller.py \ +# --config examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.yaml +defaults: ./dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml +token_capture: + enabled: true +policy: + generation: + vllm_cfg: + expose_http_server: true +# Route the DAPO math task through NeMo-Gym for capture (OpenMathReasoning via +# math_with_judge, HF-downloadable so CI can ng_prepare_data it). Judge off: +# math-verify alone scores the rows, keeping the reward binary. +env: + _override_: true + math: + num_workers: 16 + math_verify_impl: dapo_math_verify + should_use_nemo_gym: true + nemo_gym: + config_paths: + - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml + - resources_servers/math_with_judge/configs/math_with_judge.yaml + math_with_judge: + resources_servers: + math_with_judge: + judge_model_server: + name: policy_model + should_use_judge: false + policy_model: + responses_api_models: + vllm_model: + uses_reasoning_parser: false +data: + _override_: true + max_input_seq_length: 2048 + shuffle: true + num_workers: 0 + use_multiple_dataloader: false + default: + dataset_name: NemoGymDataset + env_name: nemo_gym + processor: nemo_gym_data_processor + prompt_file: null + system_prompt_file: null + train: + env_name: nemo_gym + data_path: 3rdparty/Gym-workspace/Gym/data/math_with_judge/train.jsonl + processor: nemo_gym_data_processor + dataset_name: NemoGymDataset + prompt_file: null + system_prompt_file: null + split_validation_size: 0 + # SC has no validation (val_period=0), so this is never read; point it at the + # train rows to avoid depending on a second materialized file. + validation: + env_name: nemo_gym + data_path: 3rdparty/Gym-workspace/Gym/data/math_with_judge/train.jsonl + processor: nemo_gym_data_processor + dataset_name: NemoGymDataset + prompt_file: null + system_prompt_file: null + split_validation_size: 0 +checkpointing: + checkpoint_dir: results/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture +logger: + log_dir: logs/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture + wandb: + name: dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture diff --git a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml index da26a4cf1ab..a15210ffe6b 100644 --- a/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml +++ b/examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml @@ -5,11 +5,6 @@ # No NeMo-Gym: the filter is a property of the rollout pump, not of the # environment, so the cheaper built-in math env exercises it just as well and # needs no dataset preparation step in CI. -# -# Non-capture on purpose. Setup rejects dynamic sampling together with -# token_capture.enabled, because the capture path cannot borrow a finished -# group from a later step and every drop would stall its step for a fresh -# rollout. # Launch with: # uv run examples/run_grpo_single_controller.py \ # --config examples/configs/recipes/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.yaml diff --git a/nemo_rl/algorithms/single_controller.py b/nemo_rl/algorithms/single_controller.py index 0163f77d4e6..416c480e088 100644 --- a/nemo_rl/algorithms/single_controller.py +++ b/nemo_rl/algorithms/single_controller.py @@ -147,6 +147,10 @@ _register_single_controller_partitions, ) from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + FinalizeOutcome, + RejectedGroup, + ZeroVarianceGroup, aggregate_step_metrics, reduce_advantage_pump_metrics, ) @@ -178,7 +182,6 @@ EnvironmentCheckpoint, ) from nemo_rl.environments.nemo_gym import should_use_nemo_gym -from nemo_rl.experience.dynamic_sampling import rewards_carry_no_signal from nemo_rl.experience.failures import RolloutStall from nemo_rl.experience.payload import VIOLATION_TAG_KEYS from nemo_rl.experience.reward_penalties import ( @@ -769,6 +772,8 @@ def __init__( ) # Prompts a step has drawn beyond the batch it was admitted with. self._filtered_prompts_by_step: dict[int, int] = {} + # Filtered spares promised to groups that are still finalizing. + self._filtered_spares_claimed: int = 0 # Set once the dataloader has nothing left, so waiters stop waiting. self._dataloader_exhausted: bool = False @@ -1330,43 +1335,86 @@ async def _cleanup_known_finalization_request( ) as cut: await self._cleanup_known_finalization_request_unlocked(cut, request) - async def _discard_committed_group(self, request: "ReassemblyRequest") -> bool: - """Release a committed group the controller declines to train on. + def _valid_fraction_counts( + self, finalized: "FinalizedGroup" + ) -> Optional[tuple[int, int]]: + """Return (valid, total) when the group falls below the valid-fraction bar.""" + min_valid_fraction = ( + self._master_config.token_capture.min_valid_fraction_per_group + ) + if min_valid_fraction is None: + return None + if self._master_config.token_capture.forest_training: + # Compaction expands owners into unequal row counts; use the + # finalizer's owner totals. + valid_count = int(finalized.metrics["finalize/valid_logical_owner_count"]) + total_count = int(finalized.metrics["finalize/logical_owner_count"]) + else: + valid_count = finalized.valid_row_count + total_count = finalized.total_row_count + if total_count > 0 and valid_count / total_count < min_valid_fraction: + return valid_count, total_count + return None - Returns: - True when the group was removed and its rows cleared. False when the train - pump claimed it for training before this ran, in which case the step is not - short and nothing was cleared. + @contextlib.asynccontextmanager + async def _claim_filtered_spare(self) -> AsyncIterator[None]: + """Promise a finalizing group a filtered spare, in case it is rejected. + + A rejected group's spare is taken inside the finalize cut, which cannot wait + for the pump to refill an empty pool. Claiming first waits until the pool + holds a spare no other finalizing group has claimed. A claim only counts + against the pool; the spare is popped inside the cut, and only on rejection. + + Once the dataloader is exhausted, no group waits for another spare: + rejection takes whatever remains or closes its step short. """ - # Remove the ready slot before clearing rows; a missing slot means training owns it. + if not self._dynamic_sampling_enabled: + yield + return + claimed = False + while True: + # No await between the check and the claim, so two groups cannot claim one spare. + if len(self._rollout_runtime.filtered_reserve) > ( + self._filtered_spares_claimed + ): + self._filtered_spares_claimed += 1 + claimed = True + break + if self._dataloader_exhausted: + break + await asyncio.sleep(0.1) try: - await self._buffer.remove_group(request.group_id) - except ValueError: - print( - f" finalize: group {request.group_id} was claimed for training " - "before it could be discarded; its rows belong to that step now", - flush=True, - ) - return False - await self._cleanup_known_finalization_request(request) - # The group left the buffer unconsumed, so the post-step release for - # consumed groups never reaches its pending metrics and observations. - self._rollout_runtime.finalizer_metrics_by_group.pop(request.group_id, None) - self._rollout_runtime.finalizer_rewards_by_group.pop(request.group_id, None) - return True + yield + finally: + if claimed: + self._filtered_spares_claimed -= 1 async def _finalize_with_actor( - self, request: "ReassemblyRequest", rollout_metrics: dict[str, Any] - ) -> Optional["FinalizedGroup"]: + self, + request: "ReassemblyRequest", + rollout_metrics: dict[str, Any], + *, + target_step: Optional[int], + replacements_used: int, + ) -> FinalizeOutcome: """Finalize and index one group atomically with respect to TQ saves. Returns the committed FinalizedGroup once the group is committed to - the replay buffer (callers use logical-owner counts for forests, or row - counts for ordinary capture, to decide whether to keep it), or None when - the finalizer itself dropped it as a structural outcome (ownership - already cleaned up; the caller credits the step short). A low - valid-rollout fraction is no longer a finalizer-side drop -- the caller - decides that, since only the caller can source a replacement. + the replay buffer, or None when the finalizer itself dropped it as a + structural outcome (ownership already cleaned up; the caller credits + the step short). + + A group below ``token_capture.min_valid_fraction_per_group``, or with + dynamic sampling on one whose voting rewards carry no signal, is + rejected before ``commit_finalized`` marks its slot ready, so the + sampler can never select it. Its cleanup, the spare prompt that + replaces it (or the step's shortfall credit), and that spare's ledger + reservation share the commit's mutation cut, so no snapshot sees the + group gone without its replacement. Returns a ``BelowValidFractionGroup`` + or ``ZeroVarianceGroup`` in that case. A zero-variance group takes its + spare from the filtered pool, which cannot be waited on inside the cut: + the caller holds ``_claim_filtered_spare`` across this call so the pool + is not empty when the group is rejected. """ self._finalizer_waiters += 1 queue_depth = max( @@ -1387,6 +1435,7 @@ async def _finalize_with_actor( finalize_start = time.perf_counter() rpc_submitted = False actor_reusable = False + rejected: Optional[RejectedGroup] = None try: # The actor publishes canonical rows before returning metadata. Keep # the remote write, local replay-index update, and lineage hand-off in @@ -1416,6 +1465,13 @@ async def _finalize_with_actor( else: actor_reusable = True + zero_variance = ( + not finalized.dropped + and self._rollout_manager._is_zero_variance_finalized(finalized) + ) + # The committed FinalizedGroup carries the verdict back to the + # dispatch loop so it is classified once, not recomputed there. + finalized.zero_variance = zero_variance if finalized.dropped: try: await self._cleanup_known_finalization_request_unlocked( @@ -1446,6 +1502,53 @@ async def _finalize_with_actor( "finalizer returned no metadata for non-dropped group " f"{request.group_id}" ) + elif (counts := self._valid_fraction_counts(finalized)) is not None: + try: + await self._cleanup_known_finalization_request_unlocked( + cut, request + ) + except BaseException as cleanup_error: + raise RuntimeError( + "finalizer group fell below " + "min_valid_fraction_per_group and known-key cleanup " + f"failed for group {request.group_id}" + ) from cleanup_error + replacement = self._take_replacement(replacements_used) + lineage_group_id, replacement_step = self._substitute_dropped_group( + cut, replacement, target_step + ) + rejected = BelowValidFractionGroup( + replacement=replacement, + lineage_group_id=lineage_group_id, + target_step=replacement_step, + reward_observations=list(finalized.reward_observations), + zero_variance=zero_variance, + valid_count=counts[0], + total_count=counts[1], + ) + committed = False + elif self._dynamic_sampling_enabled and zero_variance: + try: + await self._cleanup_known_finalization_request_unlocked( + cut, request + ) + except BaseException as cleanup_error: + raise RuntimeError( + "finalizer group carried no reward signal and known-key " + f"cleanup failed for group {request.group_id}" + ) from cleanup_error + replacement = self._pop_filtered_replacement(target_step) + lineage_group_id, replacement_step = self._substitute_dropped_group( + cut, replacement, target_step + ) + rejected = ZeroVarianceGroup( + replacement=replacement, + lineage_group_id=lineage_group_id, + target_step=replacement_step, + reward_observations=list(finalized.reward_observations), + zero_variance=True, + ) + committed = False else: extra_env_info: dict[str, Any] = {} if request.attribution_key is not None: @@ -1532,7 +1635,7 @@ async def _finalize_with_actor( self._available_finalizers.put_nowait(actor) finalize_total_ms = (time.perf_counter() - finalize_start) * 1000.0 if not committed: - return None + return rejected finalized.metrics.update( { "finalize/queue_wait_ms": queue_wait_ms, @@ -1752,9 +1855,13 @@ async def _dispatch_one_prompt( self._credit_shortfall(target_step) return request, rollout_metrics = captured - finalized = await self._finalize_with_actor( - request, rollout_metrics - ) + async with self._claim_filtered_spare(): + finalized = await self._finalize_with_actor( + request, + rollout_metrics, + target_step=target_step, + replacements_used=replacements, + ) if finalized is None: # Finalizer dropped the group as a structural # outcome (e.g. router replay with no routed @@ -1765,75 +1872,57 @@ async def _dispatch_one_prompt( self._buffer_capacity.release() self._credit_shortfall(target_step) return - # No per-row masking flags here, so this count is metric-only. - capture_rewards = [ - row.reward for row in finalized.reward_observations - ] self._rollout_manager.record_group_rewards( target_step, - capture_rewards, - replacement_index=replacements, - zero_variance=rewards_carry_no_signal(capture_rewards), + [row.reward for row in finalized.reward_observations], + replacement_index=replacements + filtered_replacements, + zero_variance=finalized.zero_variance, ) - min_valid_fraction = self._master_config.token_capture.min_valid_fraction_per_group - valid_count = total_count = 0 - below_threshold = False - if min_valid_fraction is not None: - if self._master_config.token_capture.forest_training: - # Compaction expands owners into unequal row - # counts; use the finalizer's owner totals. - valid_count = int( - finalized.metrics[ - "finalize/valid_logical_owner_count" - ] - ) - total_count = int( - finalized.metrics[ - "finalize/logical_owner_count" - ] - ) - else: - valid_count = finalized.valid_row_count - total_count = finalized.total_row_count - below_threshold = ( - total_count > 0 - and valid_count / total_count < min_valid_fraction - ) - if not below_threshold: + if not isinstance(finalized, RejectedGroup): self._rollout_manager.stats.committed += 1 ownership_transferred = True break - # Only the controller can replace a published group below the training threshold. - try: - discarded = await self._discard_committed_group(request) - except BaseException as cleanup_error: - raise RuntimeError( - "finalizer group fell below " - "min_valid_fraction_per_group and " - "known-key cleanup failed for group " - f"{request.group_id}" - ) from cleanup_error - if not discarded: - # Training already owns the group and its permit; do not replace it. - self._rollout_manager.stats.committed += 1 - ownership_transferred = True - break - print( - f" finalize: group {request.group_id} below " - "min_valid_fraction_per_group " - f"({valid_count}/{total_count} < " - f"{min_valid_fraction}); seeking a replacement", - flush=True, - ) - replacement = self._take_replacement(replacements) - if replacement is None: - self._rollout_manager.record_finalizer_dropped_prompt() - self._buffer_capacity.release() - self._credit_shortfall(target_step) - return - replacements += 1 - prompt = replacement - lineage_group_id = None + # The finalize cut already cleaned the group up and + # took its replacement (or credited the step short). + if isinstance(finalized, ZeroVarianceGroup): + self._rollout_manager.stats.zero_variance_groups += 1 + print( + f" dynamic sampling: dropped group " + f"{request.group_id} (target_step={target_step}), " + "its voting rewards carry no signal; " + + ( + "substituting a spare prompt" + if finalized.replacement is not None + else "no replacement; closing the step short" + ), + flush=True, + ) + if finalized.replacement is None: + self._buffer_capacity.release() + return + filtered_replacements += 1 + else: + # Too few rollouts verified to be worth training on. + print( + f" finalize: group {request.group_id} below " + "min_valid_fraction_per_group " + f"({finalized.valid_count}/{finalized.total_count} < " + f"{self._master_config.token_capture.min_valid_fraction_per_group}); " + + ( + "substituting a spare prompt" + if finalized.replacement is not None + else "no replacement; dropping the group" + ), + flush=True, + ) + if finalized.replacement is None: + self._rollout_manager.record_finalizer_dropped_prompt() + self._buffer_capacity.release() + return + replacements += 1 + prompt = finalized.replacement + lineage_group_id = finalized.lineage_group_id + target_step = finalized.target_step # A substitution is a fresh rollout, not a # continuation of the one that fell short, so it # observes the same pause a first dispatch does @@ -1924,19 +2013,11 @@ async def _dispatch_one_prompt( # a replacement, or crediting the target step short must be # one checkpoint-atomic decision. self._discard_prompt_group(cut, lineage_group_id) - if replacement is not None: - # Attempted only now that a spare is in hand, because - # the borrow is a debt and the spare is what repays it. - # Borrowing without one would leave the lender short - # instead: the same hole, one step later. - lender_step = self._promote_into_step(target_step) - if lender_step is not None: - target_step = lender_step - lineage_group_id = self._reserve_prompt_group( - cut, replacement, target_step=target_step + lineage_group_id, target_step = ( + self._substitute_dropped_group( + cut, replacement, target_step ) - else: - self._credit_shortfall(target_step) + ) if replacement is None: # Nothing was committed, so the train pump will never see this # group and never release its permit on our behalf. @@ -2461,6 +2542,46 @@ def _promote_into_step(self, target_step: Optional[int]) -> Optional[int]: ) return lender_step + def _substitute_dropped_group( + self, + cut: Optional[DataPlaneMutationCut], + replacement: Optional[DatumSpec], + target_step: Optional[int], + ) -> tuple[Optional[str], Optional[int]]: + """Stand a spare prompt in for a dropped group, or close its step short. + + Call inside the cut that removed the dropped group, so no snapshot sees the + group gone without its replacement. A finished group from a later step fills + the dropped step when one exists, and the spare is then generated for that + lender. + + Args: + cut: The cut the group was removed in; ``None`` when nothing is recorded. + replacement: The spare already taken from its pool, or ``None`` if there + was none, which credits ``target_step`` short. + target_step: The step the dropped group was stamped for. + + Returns: + The ledger ID reserved for the spare (``None`` without rollout recovery + or without a spare) and the step the spare is generated for. + """ + if replacement is None: + self._credit_shortfall(target_step) + return None, target_step + # Attempted only now that a spare is in hand, because the borrow is a debt + # and the spare is what repays it. Borrowing without one would leave the + # lender short instead: the same hole, one step later. + lender_step = self._promote_into_step(target_step) + if lender_step is not None: + target_step = lender_step + lineage_group_id = None + if self._rollout_recovery_enabled: + assert cut is not None + lineage_group_id = self._rollout_manager.reserve_prompt_group( + cut, replacement, target_step=target_step + ) + return lineage_group_id, target_step + def _credit_shortfall(self, target_step: Optional[int]) -> None: """Record that a stamped step will never receive a group it is waiting for.""" if target_step is None: diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index b64a0ff4373..cb9c6e58699 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -1552,14 +1552,6 @@ def _validate_algo_settings(master_config: MasterConfig) -> None: f"grpo.use_dynamic_sampling requires grpo.{key} to be 0 or null: " "nonzero message-level overrides can train constant-reward groups." ) - # Capture cannot borrow future groups, so routine filtering would stall training. - if master_config.token_capture.enabled: - raise ValueError( - f"{algo_prefix}.use_dynamic_sampling is not supported together with " - "token_capture.enabled: on the capture path a replacement cannot borrow " - "a finished group from a later step, so every dropped group stalls its " - "step for a full fresh rollout. Disable one of the two." - ) if grpo_cfg.dynamic_sampling_max_gen_batches < 1: raise ValueError( f"{algo_prefix}.dynamic_sampling_max_gen_batches must be >= 1 when " diff --git a/nemo_rl/algorithms/single_controller_utils/utils.py b/nemo_rl/algorithms/single_controller_utils/utils.py index b3d73d18c9b..6539ce2a27c 100644 --- a/nemo_rl/algorithms/single_controller_utils/utils.py +++ b/nemo_rl/algorithms/single_controller_utils/utils.py @@ -17,7 +17,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any, Optional import numpy as np import torch @@ -26,6 +26,11 @@ from nemo_rl.algorithms.metric_utils import REWARD_KEY from nemo_rl.data_plane import KVBatchMeta +if TYPE_CHECKING: + from nemo_rl.data.interfaces import DatumSpec + from nemo_rl.experience.reward_penalties import FinalizedReward + from nemo_rl.experience.rollout_reassembler import FinalizedGroup + # Reduction rules for all_mb_metrics. Mirror grpo.py / grpo_sync.py. _MB_METRIC_MIN: frozenset[str] = frozenset( {"probs_ratio_min", "probs_ratio_clamped_min", "opd_full_reverse_kl_min"} @@ -177,6 +182,48 @@ def from_values(cls, values: torch.Tensor) -> AdvantagePartial: ) +@dataclass(frozen=True) +class RejectedGroup: + """A finalized group the controller rejected before commit. + + The group never became selectable. ``replacement`` is the spare prompt taken + in the same mutation cut that cleaned the group up (None when the step was + credited short instead), ``target_step`` the step it was reserved for, and + ``lineage_group_id`` its ledger reservation when rollout recovery is enabled. + ``reward_observations`` are the valid rows' rewards and ``zero_variance`` + whether the rows that vote carried no signal; the pump still tallies both + into the step's reward metrics. + """ + + replacement: Optional[DatumSpec] + lineage_group_id: Optional[str] + target_step: Optional[int] + reward_observations: list[FinalizedReward] + zero_variance: bool + + +@dataclass(frozen=True) +class BelowValidFractionGroup(RejectedGroup): + """Rejected by ``token_capture.min_valid_fraction_per_group``.""" + + valid_count: int + total_count: int + + +@dataclass(frozen=True) +class ZeroVarianceGroup(RejectedGroup): + """Rejected by dynamic sampling: the voting rewards left every row zero advantage. + + ``target_step`` is the lender's when the dropped step was filled from a later + step's finished group, which the replacement then repays. + """ + + +type FinalizeOutcome = FinalizedGroup | RejectedGroup | None +"""What ``_finalize_with_actor`` returns: the group committed, a group the controller +rejected, or ``None`` when the finalizer dropped it as a structural outcome.""" + + def reduce_advantage_pump_metrics( reward_partials: list[RewardPartial], advantage_partials: list[AdvantagePartial], diff --git a/nemo_rl/experience/dynamic_sampling.py b/nemo_rl/experience/dynamic_sampling.py index 739986fa540..b2490294996 100644 --- a/nemo_rl/experience/dynamic_sampling.py +++ b/nemo_rl/experience/dynamic_sampling.py @@ -16,11 +16,14 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from nemo_rl.data_plane.schema import MASK_SAMPLE from nemo_rl.experience.interfaces import PromptGroupRecord +if TYPE_CHECKING: + from nemo_rl.experience.rollout_reassembler import FinalizedGroup + def mask_sample_flagged(extra: dict[str, Any] | None) -> bool: """Read both mask flags, since Gym may default the first-class flag to false.""" @@ -32,9 +35,9 @@ def mask_sample_flagged(extra: dict[str, Any] | None) -> bool: def rewards_carry_no_signal(rewards: Sequence[float]) -> bool: - """Whether a group's participating rewards leave every row zero advantage. + """Whether a group's voting rewards leave every row zero advantage. - True for at most one participating reward, or when they are all exactly equal. + True for at most one voting reward, or when they are all exactly equal. Exact equality rather than a std threshold because float roundoff can give a uniform group a tiny nonzero std. @@ -47,13 +50,16 @@ def rewards_carry_no_signal(rewards: Sequence[float]) -> bool: return len(rewards) <= 1 or max(rewards) == min(rewards) -def participating_rewards( - record: PromptGroupRecord, +def voting_rewards( + rewards: Sequence[float], *, + mask_sample: Sequence[bool], + truncated: Sequence[bool], + loss_multiplier: float, masked_reward_policy: Optional[str], overlong_filtering: bool, ) -> list[float]: - """The rewards that vote in this group's baseline and std. + """The rewards that vote in a group's baseline and std. Mirrors v1's ``_dynamic_sampling_valid_mask``, which is the mask the group statistics are actually computed under. Filtering on the full reward list instead @@ -70,20 +76,71 @@ def participating_rewards( ``overlong_filtering`` so do truncated ones -- the rows the loss drops later in the step. Under ``include`` both still vote: their rewards are measurements of the policy even though they carry no loss. + + Args: + rewards: One Gym reward per row, before effort shaping and reward penalties. + mask_sample: Per-row environment ``mask_sample`` flag, parallel to ``rewards``. + truncated: Per-row truncation flag, parallel to ``rewards``. + loss_multiplier: The group's dataset loss weight. + masked_reward_policy: ``grpo.masked_reward_policy``; ``None`` is PPO, which + has no group-relative baseline to mask rows out of. + overlong_filtering: ``grpo.overlong_filtering``. """ - if record.loss_multiplier == 0: + if loss_multiplier == 0: return [] - # None is PPO: no group-relative baseline exists to mask rows out of. if masked_reward_policy != "exclude": - return [completion.reward for completion in record.completions] + return list(rewards) return [ - completion.reward - for completion in record.completions - if not mask_sample_flagged(completion.env_extras) - and not (overlong_filtering and completion.truncated) + reward + for reward, masked, is_truncated in zip( + rewards, mask_sample, truncated, strict=True + ) + if not masked and not (overlong_filtering and is_truncated) ] +def voting_rewards_from_record( + record: PromptGroupRecord, + *, + masked_reward_policy: Optional[str], + overlong_filtering: bool, +) -> list[float]: + """The rewards that vote in a group's baseline, read from its ``Completion`` rows.""" + return voting_rewards( + [ + completion.reward + if completion.reward_original is None + else completion.reward_original + for completion in record.completions + ], + mask_sample=[ + mask_sample_flagged(completion.env_extras) + for completion in record.completions + ], + truncated=[completion.truncated for completion in record.completions], + loss_multiplier=record.loss_multiplier, + masked_reward_policy=masked_reward_policy, + overlong_filtering=overlong_filtering, + ) + + +def voting_rewards_from_finalized( + finalized: "FinalizedGroup", + *, + masked_reward_policy: Optional[str], + overlong_filtering: bool, +) -> list[float]: + """The rewards that vote in a group's baseline, read from a finalized capture group.""" + return voting_rewards( + finalized.reward_original, + mask_sample=finalized.reward_mask_sample, + truncated=finalized.reward_truncated, + loss_multiplier=finalized.loss_multiplier, + masked_reward_policy=masked_reward_policy, + overlong_filtering=overlong_filtering, + ) + + @dataclass class StepRewardStats: """Reward totals for the groups generated for one training step. diff --git a/nemo_rl/experience/interfaces.py b/nemo_rl/experience/interfaces.py index 29320e93f2b..72751edf0fa 100644 --- a/nemo_rl/experience/interfaces.py +++ b/nemo_rl/experience/interfaces.py @@ -49,6 +49,9 @@ class Completion: env_extras: Optional[dict[str, Any]] truncated: bool reward: float + # The environment's reward before effort shaping and reward penalties, which + # dynamic sampling filters on. ``None`` means nothing rewrote ``reward``. + reward_original: Optional[float] = None @dataclass diff --git a/nemo_rl/experience/rollout_manager.py b/nemo_rl/experience/rollout_manager.py index 8f0168f20f1..008dfea3cd6 100644 --- a/nemo_rl/experience/rollout_manager.py +++ b/nemo_rl/experience/rollout_manager.py @@ -52,9 +52,10 @@ StepRewardStats, drain_step_reward_metrics, mask_sample_flagged, - participating_rewards, record_group_rewards, rewards_carry_no_signal, + voting_rewards_from_finalized, + voting_rewards_from_record, ) from nemo_rl.experience.failures import ( FailureClass, @@ -123,6 +124,7 @@ if TYPE_CHECKING: from nemo_rl.algorithms.single_controller_utils.config import RolloutRecoveryConfig + from nemo_rl.experience.rollout_reassembler import FinalizedGroup from nemo_rl.experience.rollout_reassembler_actor import ReassemblyRequest @@ -1534,6 +1536,8 @@ async def consume(pending_rows: list[dict], refusal_streak: int = 0) -> None: else None, ) else: + # Effort shaping and the penalties rewrite this reward in place. + result["reward_original"] = float(result["full_result"]["reward"]) shaping_by_rowidx[rowidx] = _apply_effort_shaping( [result], [inputs_by_rowidx[rowidx]], self._effort_config ) @@ -1926,6 +1930,7 @@ def completion_extras(full_result: dict) -> dict: env_extras=env_extras, truncated=truncated, reward=float(result["full_result"]["reward"]), + reward_original=result.get("reward_original"), ) ) return completions, penalty_counts @@ -2500,17 +2505,23 @@ async def run_rollout( dispatch_recorder=dispatch_recorder, ) - def _participating_rewards(self, record: PromptGroupRecord) -> list[float]: + def _voting_rewards(self, record: PromptGroupRecord) -> list[float]: """This group's voting rewards under the run's masking policy.""" - return participating_rewards( + return voting_rewards_from_record( record, masked_reward_policy=self._masked_reward_policy, overlong_filtering=self._overlong_filtering, ) - def _is_zero_variance_group(self, record: PromptGroupRecord) -> bool: - """Whether this group's voters leave every row zero advantage.""" - return rewards_carry_no_signal(self._participating_rewards(record)) + def _is_zero_variance_finalized(self, finalized: "FinalizedGroup") -> bool: + """Whether a finalized capture group's voters leave every row zero advantage.""" + return rewards_carry_no_signal( + voting_rewards_from_finalized( + finalized, + masked_reward_policy=self._masked_reward_policy, + overlong_filtering=self._overlong_filtering, + ) + ) def record_group_rewards( self, @@ -2667,16 +2678,15 @@ async def generate_and_push( finally: if inflight_registry is not None: inflight_registry.pop(tq_group_id, None) - if self._filter_zero_variance_groups and self._is_zero_variance_group( - record - ): + voting = self._voting_rewards(record) + is_zero_variance = rewards_carry_no_signal(voting) + if self._filter_zero_variance_groups and is_zero_variance: # Filtering happens before commit, so removing the reservation writes no rows. await self._tq_buffer.remove_group(tq_group_id) # A filtered group is a healthy rollout and breaks a consecutive infrastructure-failure run. self._consecutive_infra_drops = 0 self._stats.zero_variance_groups += 1 rewards = [c.reward for c in record.completions] - voting = self._participating_rewards(record) self.record_group_rewards( target_step, rewards, @@ -2706,7 +2716,7 @@ async def generate_and_push( target_step, [c.reward for c in record.completions], replacement_index=replacement_index, - zero_variance=self._is_zero_variance_group(record), + zero_variance=is_zero_variance, ) except Exception as error: # A failed rollout must not leave an unready slot that can block an diff --git a/nemo_rl/experience/rollout_reassembler.py b/nemo_rl/experience/rollout_reassembler.py index 97cef6b5098..bd3c2e906fa 100644 --- a/nemo_rl/experience/rollout_reassembler.py +++ b/nemo_rl/experience/rollout_reassembler.py @@ -196,6 +196,14 @@ class FinalizedGroup: canonical_output_tokens: int = 0 metrics: dict[str, float] = field(default_factory=dict) reward_observations: list[FinalizedReward] = field(default_factory=list) + # Parallel to ``reward_observations``: each row's environment ``mask_sample`` + # flag, truncation and original reward (before effort shaping and penalties), plus + # the group's dataset loss weight. Which rewards vote in the group baseline + # depends on controller-side policy, so the raw flags travel instead of a verdict. + reward_mask_sample: list[bool] = field(default_factory=list) + reward_truncated: list[bool] = field(default_factory=list) + reward_original: list[float] = field(default_factory=list) + loss_multiplier: float = 1.0 # True when the finalizer rejected the whole group as a structural outcome # (see drop_reason); the caller aborts the slot instead of committing it. # Policy decisions like a low valid-rollout fraction are made by the caller, @@ -215,6 +223,12 @@ class FinalizedGroup: # Each trained call's (model call ID, weight version it began on, tokens it # generated), across the group's verified rollouts; empty when dropped. call_versions: list[tuple[str, int, int]] = field(default_factory=list) + # The controller-set zero-variance verdict. The reassembler ships the raw + # reward flags above; the controller classifies the group once (via + # RolloutManager, which owns the voting settings) and fills this in before + # returning the committed group, so the dispatch loop reads it instead of + # reclassifying. + zero_variance: bool = False def _capture_token_metrics( @@ -1092,6 +1106,13 @@ def finalize_group( # termination rates like the penalty counts; receipt-side rates only # saw a placeholder truncated=False. truncated_flags = self._truncated_flags(rows, logical) + reward_mask_sample = [ + mask_sample[index] for index, _, row in reward_rows if row.valid + ] + reward_truncated = [ + truncated_flags[index] for index, _, row in reward_rows if row.valid + ] + reward_original = [rewards[slot] for _, slot, row in reward_rows if row.valid] metrics["finalize/truncated_count"] = 0.0 for index, slot, row in reward_rows: if row.valid: @@ -1121,6 +1142,9 @@ def finalize_group( metrics=metrics, rollouts_ms=_rollouts_ms, reward_observations=reward_observations, + reward_mask_sample=reward_mask_sample, + reward_truncated=reward_truncated, + reward_original=reward_original, logical=logical, execution_row_multiple=execution_row_multiple, canonical_sample_ids=canonical_sample_ids, @@ -1154,6 +1178,9 @@ def _publish_rows( metrics: dict[str, float], rollouts_ms: float, reward_observations: list[FinalizedReward], + reward_mask_sample: list[bool], + reward_truncated: list[bool], + reward_original: list[float], logical: Optional[_LogicalRows], execution_row_multiple: int, canonical_sample_ids: list[str], @@ -1425,6 +1452,10 @@ def _publish_rows( ), metrics=metrics, reward_observations=reward_observations, + reward_mask_sample=reward_mask_sample, + reward_truncated=reward_truncated, + reward_original=reward_original, + loss_multiplier=loss_multiplier, valid_row_count=len(valid_rows), total_row_count=real_row_count, ) diff --git a/tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh b/tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh new file mode 100755 index 00000000000..79c7c7e3e27 --- /dev/null +++ b/tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=4 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=5 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +# ~25 min startup (30B-MoE load + CUDA-graph warmup + nemo_gym servers) plus +# ~17 min/step on OpenMathReasoning, and dynamic sampling generates extra groups +# to replace the filtered ones. Five steps cover a refit with requests in flight +# and a checkpoint save through the finalizer. Calibrated against job 21912385 +# (six steps in ~115 min including startup). +NUM_MINUTES=130 # Initial 4-node budget; calibrate against a completed run. +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +cd $PROJECT_ROOT + +# Prepare the NeMo-Gym math_with_judge (OpenMathReasoning) taskset. math_with_judge +# downloads from the HF repo nvidia/Nemotron-RL-math-OpenMathReasoning, so CI can +# materialize it; the recipe reads the train rows written here. Mirrors the +# workplace_assistant prep in grpo_async_gym.sh but needs no tool trimming. +cd 3rdparty/Gym-workspace/Gym +if [[ ! -f env.yaml ]]; then + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "[ERROR] HF_TOKEN is not set" + exit 1 + fi + echo "hf_token: $HF_TOKEN" >> env.yaml +fi +uv run ng_prepare_data "+config_paths=[resources_servers/math_with_judge/configs/math_with_judge.yaml]" \ + +output_dirpath=data/math_with_judge \ + +mode=train_preparation \ + +should_download=true \ + +data_source=huggingface +cd - + +# Token capture is only wired up on the SingleController path. +uv run examples/run_grpo_single_controller.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + checkpointing.save_period=3 \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Gates follow the non-capture DAPO single-controller dynamic-sampling nightly +# (dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.sh, #4588), +# plus the capture-only finalize/* health gates this recipe adds. +# +# The dynamic-sampling gates: the filter has to be doing something (zero-variance +# groups accumulate across the run) and has to be paying for it without shrinking the +# step (skipped_step and dropped_prompt_groups stay 0 -- the batch size the feature +# exists to hold). unfiltered_reward, not reward, is the progress signal: reward here +# is the mean over the trained (kept) rows, which excludes the all-same-reward groups, +# so it is the wrong denominator to gate progress on (v1 reports the two under swapped +# names; see docs/guides/single-controller.md#dynamic-sampling). +# +# The capture gates: a nonzero finalize/invalid_row_rate or capture_poisoned_rollouts +# means capture or recovery lost a rollout's tokens. +# +# Thresholds are calibrated on job 21912385 (Lightning-30B, Gym OpenMathReasoning, +# vLLM generation, 4n8g), reading its six completed steps: unfiltered_reward 0.21-0.29, +# per-step zero-variance groups 10-27 (>=20 cumulative over five steps with margin), +# token_mult_prob_error median 1.018 / max 1.029 (looser than #4588's built-in-math +# bounds -- Gym rollouts through the OpenAI server drift slightly more), total_step_time +# ~1040s. No max(total_step_time) gate because save_period=3 bundles a checkpoint write +# into one step; median excludes it. +uv run tests/check_metrics.py $JSON_METRICS \ + "len(data[\"train/loss\"]) == $MAX_STEPS" \ + 'all_finite(data["train/loss"])' \ + 'all_finite(data["train/token_mult_prob_error"])' \ + 'median(data["train/token_mult_prob_error"]) < 1.03' \ + 'max(data["train/token_mult_prob_error"]) < 1.05' \ + 'median(data["train/unfiltered_reward"]) > 0.1' \ + 'max(data["train/finalize/invalid_row_rate"]) == 0' \ + 'max(data["train/finalize/capture_poisoned_rollouts"]) == 0' \ + 'max(data["rollout/zero_variance_groups_total"]) >= 20' \ + 'min(data["train/skipped_step"]) == max(data["train/skipped_step"]) == 0' \ + 'min(data["train/dropped_prompt_groups"]) == max(data["train/dropped_prompt_groups"]) == 0' \ + 'median(data["timing/train/total_step_time"]) < 1500' + +# Clean up checkpoint directory after successful run to save space. +rm -rf "$CKPT_DIR" diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index ec4c41c06cb..e6b3e6395a2 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -215,6 +215,7 @@ tests/test_suites/llm/grpo-qwen3-8b-base-1n8g-megatron-lora-tq_nixl.sh # Single Controller (SC) tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-single-controller.sh +tests/test_suites/llm/dapo-nanov3.5-30BA3B-4n8g-megatron-vllm-noncolocated-async-gym-single-controller-token_capture.sh tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-async-1off-single-controller-streaming2.sh tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron-single-controller-sync.sh tests/test_suites/llm/grpo-qwen2.5-math-1.5b-instruct-1n8g-megatron_generation-noncolocated-single-controller-async-lag4.sh diff --git a/tests/unit/algorithms/test_advantage_validity.py b/tests/unit/algorithms/test_advantage_validity.py index 1d7724a508d..eed66044604 100644 --- a/tests/unit/algorithms/test_advantage_validity.py +++ b/tests/unit/algorithms/test_advantage_validity.py @@ -187,7 +187,7 @@ def test_reinforce_pp_none_valid_mask_keeps_legacy_all_valid_behavior(): @pytest.mark.parametrize("include", [False, True]) @pytest.mark.parametrize("leave_one_out", [False, True]) @pytest.mark.parametrize("normalize", [False, True]) -def test_gdpo_statistics_match_participating_rewards(include, leave_one_out, normalize): +def test_gdpo_statistics_match_voting_rewards(include, leave_one_out, normalize): """Check actual final values, including weighted components and whitening.""" prompt_ids = torch.tensor([[0]] * 4 + [[1]] * 4) loss_weights = torch.tensor([1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0]) diff --git a/tests/unit/experience/test_rollout_generation_failures.py b/tests/unit/experience/test_rollout_generation_failures.py index 3cf0f9bd308..adf94926e48 100644 --- a/tests/unit/experience/test_rollout_generation_failures.py +++ b/tests/unit/experience/test_rollout_generation_failures.py @@ -536,7 +536,11 @@ async def _result(rowidx): return ( rowidx, {"name": "agent"}, - {"input_message_log": [], "message_log": []}, + { + "input_message_log": [], + "message_log": [], + "full_result": {"reward": 1.0}, + }, None, ) diff --git a/tests/unit/experience/test_rollout_manager.py b/tests/unit/experience/test_rollout_manager.py index f7c5825e2f7..2544ddf412a 100644 --- a/tests/unit/experience/test_rollout_manager.py +++ b/tests/unit/experience/test_rollout_manager.py @@ -1468,10 +1468,13 @@ def test_nemo_gym_reward_penalties_match_legacy_rewards_counts_and_metrics( ): impl = _nemo_gym_impl(True, reward_penalty_config) result = _reward_penalty_result(output, assistant_overrides, assistant_tokens) + result["reward_original"] = 1.0 completions, penalty_counts = impl._results_to_completions([result]) assert completions[0].reward == 0.0 + # Dynamic sampling still sees the reward the penalty overwrote. + assert completions[0].reward_original == 1.0 assert penalty_counts[count_key] == 1 assert sum(penalty_counts.values()) == 1 assert impl._compute_reward_penalty_metrics(penalty_counts, 1) == {metric_name: 1.0} @@ -2959,7 +2962,12 @@ def _scored_record(*rewards: float) -> PromptGroupRecord: prompt=[], extra_env_info=None, metadata={}, - completions=[SimpleNamespace(reward=reward) for reward in rewards], + completions=[ + SimpleNamespace( + reward=reward, reward_original=None, env_extras=None, truncated=False + ) + for reward in rewards + ], rollout_metrics={}, ) diff --git a/tests/unit/experience/test_rollout_reassembler_actor.py b/tests/unit/experience/test_rollout_reassembler_actor.py index 8a14482c604..136ef67655f 100644 --- a/tests/unit/experience/test_rollout_reassembler_actor.py +++ b/tests/unit/experience/test_rollout_reassembler_actor.py @@ -216,6 +216,10 @@ def test_rpc_dataclass_fields_are_classified() -> None: "canonical_output_tokens", "metrics", "reward_observations", + "reward_mask_sample", + "reward_truncated", + "reward_original", + "loss_multiplier", "dropped", "drop_reason", "valid_row_count", @@ -223,6 +227,8 @@ def test_rpc_dataclass_fields_are_classified() -> None: "token_metrics", # One (model call ID, weight version, tokens) tuple per trained call. "call_versions", + # Controller-set verdict: the finalized group scored all-same reward. + "zero_variance", } diff --git a/tests/unit/experience/test_rollouts.py b/tests/unit/experience/test_rollouts.py index ad29d70bf0c..82a95023276 100644 --- a/tests/unit/experience/test_rollouts.py +++ b/tests/unit/experience/test_rollouts.py @@ -2960,7 +2960,14 @@ class _DuplicateStream: def __init__(self): self.values = iter( [ - _ReadyRef((0, {"name": "agent"}, {"value": "first"}, None)), + _ReadyRef( + ( + 0, + {"name": "agent"}, + {"value": "first", "full_result": {"reward": 1.0}}, + None, + ) + ), _ReadyRef((0, {"name": "agent"}, {"value": "duplicate"}, None)), ] ) diff --git a/tests/unit/experience/test_selected_forest_boundaries.py b/tests/unit/experience/test_selected_forest_boundaries.py index 85572d59078..f7032e69d50 100644 --- a/tests/unit/experience/test_selected_forest_boundaries.py +++ b/tests/unit/experience/test_selected_forest_boundaries.py @@ -1304,6 +1304,12 @@ def variable_completion(capture: Any, active: Any, **kwargs: Any) -> Any: assert observation.rollout_id == "attempt" assert observation.reward == pytest.approx(2.0 - (scored_index + 1) / 10) assert observation.log_context.agent_name == "test-agent" + # Raw voting flags ride beside the observation, one per observation. + assert result.reward_mask_sample == [False] + assert len(result.reward_truncated) == 1 + # The Gym reward is the dispatcher's input, before effort shaping. + assert result.reward_original == [1.0] + assert result.loss_multiplier == 1.0 assert result.metrics[f"finalize/effort/low/{scored_index + 1}"] == 1 for key in result.meta.sample_ids[:3]: assert forest.plane.rows["canonical", key][ @@ -1317,6 +1323,59 @@ def variable_completion(capture: Any, active: Any, **kwargs: Any) -> Any: ) +def test_ordinary_finalize_carries_nondefault_voting_fields( + forest: ForestCapture, monkeypatch: pytest.MonkeyPatch +) -> None: + """The finalizer ships the raw per-row voting fields the controller classifies on. + + An ordinary (non-Forest) group with a masked valid row, an invalid row and a + truncated valid row, under a non-unit dataset loss weight, must return those exact + flags, the pre-shaping Gym rewards and the loss_multiplier for the valid rows only + -- all away from their defaults -- so the controller's zero-variance filter reads + real signal, not placeholders, and the invalid row never votes. + """ + # Two independent single-call rollouts, each its own ordinary reward row. + records_a, _ = forest.capture([None], owner="attempt") + records_b, _ = forest.capture([None], owner="healthy") + + # Keep truncation mixed across the two valid reward rows. + monkeypatch.setattr( + forest.finalizer, + "_truncated_flags", + lambda rows, logical: [False, False, True], + ) + + result = forest.finalizer.finalize_group( + "group", + ["attempt", "bad", "healthy"], + [ + receipt(records_a, terminal=records_a[-1]), + # An invalid row between the two valid ones: the voting lists must skip it. + None, + # capture_key must match the rollout_id "healthy" or finalize rejects the row. + {**receipt(records_b, terminal=records_b[-1]), "capture_key": "healthy"}, + ], + [0.25, 7.0, 0.75], + mask_sample=[True, False, False], + fallback_weight_version=7, + prompt_idx=99, + loss_multiplier=0.5, + canonical_sample_ids=["group_g0", "group_g1", "group_g2"], + ) + + assert result.total_row_count == 3 + assert result.valid_row_count == 2 + assert len(result.reward_observations) == 2 + # The raw voting flags ride one-per-valid-row, in rollout order, not their + # defaults, so the controller classifies the real group. + assert result.reward_mask_sample == [True, False] + assert result.reward_truncated == [False, True] + # The Gym rewards, before any effort shaping or penalties. + assert result.reward_original == [0.25, 0.75] + # The dataset-level weight is carried through unchanged (non-unit). + assert result.loss_multiplier == 0.5 + + def test_publication_identity_drift_preserves_staging(forest, monkeypatch): records, _ = forest.capture([None, 0, None]) original = forest.finalizer._finalize_logical_rows diff --git a/tests/unit/single_controller/test_finalizer_lifecycle.py b/tests/unit/single_controller/test_finalizer_lifecycle.py index 396d8765d73..52a5cee2763 100644 --- a/tests/unit/single_controller/test_finalizer_lifecycle.py +++ b/tests/unit/single_controller/test_finalizer_lifecycle.py @@ -18,6 +18,7 @@ import asyncio import threading +from collections import deque from types import SimpleNamespace from typing import Any from unittest.mock import ANY, AsyncMock, MagicMock @@ -34,8 +35,16 @@ RolloutRuntimeState, SingleControllerActor, ) +from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + ZeroVarianceGroup, +) from nemo_rl.data_plane import KVBatchMeta from nemo_rl.data_plane.schema import ROLLOUT_METRICS, ROUTE_PLAN_TAG +from nemo_rl.experience.dynamic_sampling import ( + rewards_carry_no_signal, + voting_rewards_from_finalized, +) from nemo_rl.experience.reward_penalties import FinalizedReward from nemo_rl.experience.rollout_reassembler import FinalizedGroup from nemo_rl.experience.rollout_reassembler_actor import ReassemblyRequest @@ -116,17 +125,39 @@ def _controller(actor: object) -> Any: ctrl._rollout_recovery_ledger = MagicMock() ctrl._rollout_recovery_ledger.__contains__.return_value = False ctrl._rollout_manager = MagicMock() + # Classify with real reward helpers instead of a truthy MagicMock. + ctrl._rollout_manager._is_zero_variance_finalized.side_effect = ( + lambda finalized: rewards_carry_no_signal( + voting_rewards_from_finalized( + finalized, + masked_reward_policy=( + None if ctrl._is_ppo else ctrl._algo_cfg.masked_reward_policy + ), + overlong_filtering=ctrl._algo_cfg.overlong_filtering, + ) + ) + ) ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._buffer = MagicMock() ctrl._buffer.commit_finalized = AsyncMock() - ctrl._buffer.remove_group = AsyncMock() ctrl._dp_client = _DataPlaneClient() ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() ctrl._partition_id = "canonical" ctrl._master_config = SimpleNamespace( - token_capture=SimpleNamespace(staging_partition="staging"), - grpo=SimpleNamespace(num_prompts_per_step=1), + token_capture=SimpleNamespace( + staging_partition="staging", + min_valid_fraction_per_group=None, + forest_training=False, + ), + grpo=SimpleNamespace( + num_prompts_per_step=1, + masked_reward_policy="include", + overlong_filtering=False, + ), ) + ctrl._algo_cfg = ctrl._master_config.grpo + ctrl._is_ppo = False + ctrl._dynamic_sampling_enabled = False ctrl._trainer_version = 3 ctrl._train_steps = 3 return ctrl @@ -170,7 +201,9 @@ def test_successful_actor_finalization_returns_actor_and_transfers_ownership( ctrl = _controller(actor) request = _request(resolved_agent_name, attribution_key=attribution_key) - committed = asyncio.run(ctrl._finalize_with_actor(request, {})) + committed = asyncio.run( + ctrl._finalize_with_actor(request, {}, target_step=None, replacements_used=0) + ) assert committed is result assert finalize.calls == [request] @@ -213,7 +246,11 @@ def test_capture_rollout_metrics_ride_on_the_committed_meta() -> None: ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) metrics = {"agent/rollout_time_taken/mean": 12.5} - asyncio.run(ctrl._finalize_with_actor(_request(), metrics)) + asyncio.run( + ctrl._finalize_with_actor( + _request(), metrics, target_step=None, replacements_used=0 + ) + ) committed_meta = ctrl._buffer.commit_finalized.await_args.args[2] assert committed_meta.extra_info == {"existing": 1, ROLLOUT_METRICS: [metrics]} @@ -248,7 +285,11 @@ def test_finalizer_token_metrics_and_rollout_metrics_share_one_sidecar( result = _token_metrics_result(token_metrics) ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) - asyncio.run(ctrl._finalize_with_actor(_request(), rollout_metrics)) + asyncio.run( + ctrl._finalize_with_actor( + _request(), rollout_metrics, target_step=None, replacements_used=0 + ) + ) committed_meta = ctrl._buffer.commit_finalized.await_args.args[2] assert committed_meta.extra_info[ROLLOUT_METRICS] == [ @@ -262,7 +303,12 @@ def test_overlapping_finalizer_and_rollout_metrics_fail_the_commit() -> None: with pytest.raises(RuntimeError, match="both report"): asyncio.run( - ctrl._finalize_with_actor(_request(), {"gen_tokens_per_sample/mean": 1.0}) + ctrl._finalize_with_actor( + _request(), + {"gen_tokens_per_sample/mean": 1.0}, + target_step=None, + replacements_used=0, + ) ) ctrl._buffer.commit_finalized.assert_not_awaited() @@ -336,6 +382,8 @@ def score(received_meta: KVBatchMeta) -> None: ctrl._finalize_with_actor( _request(None if outcome == "missing_route" else "image_agent"), {}, + target_step=None, + replacements_used=0, ) ) try: @@ -401,105 +449,396 @@ def score(received_meta: KVBatchMeta) -> None: asyncio.run(exercise()) -def test_below_threshold_group_releases_pending_finalizer_state() -> None: - """A committed group dropped for min_valid_fraction_per_group leaves no state.""" - meta = KVBatchMeta( - partition_id="canonical", - task_name="train", - sample_ids=["group_g0"], - fields=["input_ids"], - sequence_lengths=[3], - tags=[{"weight_version": 3}], - ) - result = FinalizedGroup( - meta=meta, +def _below_threshold_result() -> FinalizedGroup: + # One of two rollouts verified: 0.5 < the 0.75 bar set by _below_threshold. + return FinalizedGroup( + meta=KVBatchMeta( + partition_id="canonical", + task_name="train", + sample_ids=["group_g0", "group_g1"], + fields=["input_ids"], + sequence_lengths=[3, 3], + tags=[{"weight_version": 3}, {"weight_version": 3}], + ), group_min_wv=3, group_max_wv=3, staging_keys=["group_g0/call"], - metrics={"finalize/invalid_row_rate": 0.0}, + metrics={"finalize/invalid_row_rate": 0.5}, reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + valid_row_count=1, + total_row_count=2, ) - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) - request = _request() - async def _below_threshold_branch() -> None: - # The same calls the rollout pump makes when - # valid_row_count / total_row_count < min_valid_fraction_per_group. - assert await ctrl._finalize_with_actor(request, {}) is result - assert "group" in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" in ctrl._rollout_runtime.finalizer_rewards_by_group - assert await ctrl._discard_committed_group(request) is True - asyncio.run(_below_threshold_branch()) +def _below_threshold(ctrl: Any, *, spares: list[dict[str, Any]]) -> None: + ctrl._master_config.token_capture.min_valid_fraction_per_group = 0.75 + ctrl._async_cfg = SimpleNamespace( + rollout_failure=SimpleNamespace( + on_dropped_prompt="replace", max_replacement_attempts=1 + ) + ) + ctrl._rollout_runtime.replacement_reserve = deque(spares) + ctrl._rollout_runtime.batch_shortfall = {} + ctrl._rollout_runtime.sampler_stamps_target_steps = True + ctrl._rollout_recovery_enabled = True - ctrl._buffer.remove_group.assert_awaited_once_with("group") - ctrl._buffer.abort.assert_called_once_with("group") - assert "group" not in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" not in ctrl._rollout_runtime.finalizer_rewards_by_group +class _ReadyWatchingClient(_DataPlaneClient): + """Records whether any replay slot was ready while rows were being cleared.""" -def test_discarded_committed_group_leaves_no_ready_slot_in_the_buffer() -> None: - """abort() skips ready slots, so discarding must remove the committed slot itself.""" - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize())) - buffer = TQReplayBuffer( - ctrl._dp_client, + def __init__(self) -> None: + super().__init__() + self.buffer: TQReplayBuffer | None = None + self.ready_during_clear: list[bool] = [] + + async def clear_samples(self, *, sample_ids: list[str], partition_id: str) -> None: + assert self.buffer is not None + self.ready_during_clear.append(any(self.buffer.ready_list)) + await asyncio.sleep(0) # let any other task run mid-cleanup + await super().clear_samples(sample_ids=sample_ids, partition_id=partition_id) + + +def test_below_threshold_group_never_becomes_selectable() -> None: + """A rejected group leaves the buffer without ever being marked ready.""" + enricher_calls: list[str] = [] + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[]) + client = _ReadyWatchingClient() + ctrl._dp_client = client + ctrl._buffer = TQReplayBuffer( + client, partition_id="canonical", pad_value_dict={"token_ids": 0}, include_message_violation_fields=False, require_routed_experts=False, ) - buffer.set_data_plane_checkpoint_barrier(ctrl._data_plane_checkpoint_barrier) - ctrl._buffer = buffer - request = _request() - meta = KVBatchMeta( - partition_id="canonical", - task_name=None, - sample_ids=list(request.canonical_sample_ids), - fields=None, + client.buffer = ctrl._buffer + ctrl._buffer.set_data_plane_checkpoint_barrier(ctrl._data_plane_checkpoint_barrier) + + async def _enrich(meta: KVBatchMeta, extra_env_info: Any) -> None: + enricher_calls.append(meta.sample_ids[0]) + + ctrl._buffer.set_post_write_enricher(_enrich) + ctrl._buffer.reserve(weight_version=3, target_step=3, group_id="group") + + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) + + assert outcome == BelowValidFractionGroup( + replacement=None, + lineage_group_id=None, + target_step=3, + reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + zero_variance=True, + valid_count=1, + total_count=2, + ) + assert ctrl._buffer.group_ids == () + assert client.ready_during_clear == [False, False] + assert client.clear_calls == [ + {"sample_ids": ["group_g0"], "partition_id": "canonical"}, + {"sample_ids": ["group_g0/call"], "partition_id": "staging"}, + ] + # Never committed: no enrichment, no pending metrics, and the step was + # credited short because no spare was left. + assert enricher_calls == [] + assert "group" not in ctrl._rollout_runtime.finalizer_metrics_by_group + assert "group" not in ctrl._rollout_runtime.finalizer_rewards_by_group + assert ctrl._rollout_runtime.batch_shortfall == {3: 1} + assert ctrl._available_finalizers.qsize() == 1 + + +@pytest.mark.parametrize("recovery", [True, False]) +def test_below_threshold_replacement_is_reserved_in_the_cleanup_cut( + recovery: bool, +) -> None: + """The rejected group's ledger discard and its spare's reservation share a cut. + + Without rollout recovery the spare is not reserved here: the pump never + discards a lineage it did not get, so generate_for_finalization must own it. + """ + spare = {"idx": 99} + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[spare]) + ctrl._rollout_recovery_enabled = recovery + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + ctrl._buffer.promote_ready_group = MagicMock(return_value=None) + ctrl._batch_promotions = {} + + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) ) - async def _commit_then_discard() -> None: - buffer.reserve( - weight_version=3, - group_id=request.group_id, - rollout_ids=list(request.rollout_ids), + assert outcome == BelowValidFractionGroup( + replacement=spare, + lineage_group_id="spare-group" if recovery else None, + target_step=3, + reward_observations=[FinalizedReward("group_g0", "group_g0", 1.0, None)], + zero_variance=True, + valid_count=1, + total_count=2, + ) + ctrl._buffer.commit_finalized.assert_not_awaited() + ctrl._buffer.abort.assert_called_once_with("group") + ctrl._rollout_recovery_ledger.discard_group.assert_called_once() + (discard_cut, discarded), _ = ctrl._rollout_recovery_ledger.discard_group.call_args + assert discarded == "group" + if recovery: + ctrl._rollout_manager.reserve_prompt_group.assert_called_once() + (reserve_cut, reserved), kwargs = ( + ctrl._rollout_manager.reserve_prompt_group.call_args ) - async with ctrl._data_plane_checkpoint_barrier.mutation() as cut: - await buffer.commit_finalized(cut, request.group_id, meta, 3, 3) - assert buffer.size() == 1 - assert await ctrl._discard_committed_group(request) is True + assert reserved is spare + assert kwargs == {"target_step": 3} + assert reserve_cut is discard_cut + else: + ctrl._rollout_manager.reserve_prompt_group.assert_not_called() + assert not ctrl._rollout_runtime.replacement_reserve + assert ctrl._rollout_runtime.batch_shortfall == {} + + +def test_below_threshold_replacement_borrows_a_finished_group_and_repays_the_lender() -> ( + None +): + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[{"idx": 99}]) + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + ctrl._batch_promotions = {} + ctrl._buffer.promote_ready_group = MagicMock(return_value=5) + + outcome = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) - asyncio.run(_commit_then_discard()) + ctrl._buffer.promote_ready_group.assert_called_once_with(to_target_step=3) + assert ctrl._batch_promotions == {3: 1} + # The spare is owed to the lender, so it is reserved for the lender's step. + assert outcome.target_step == 5 + assert outcome.lineage_group_id == "spare-group" + (_, reserved), kwargs = ctrl._rollout_manager.reserve_prompt_group.call_args + assert reserved == {"idx": 99} + assert kwargs == {"target_step": 5} + assert ctrl._rollout_runtime.batch_shortfall == {} + + +def _signal_result( + rewards: list[float], + *, + truncated: list[bool] | None = None, + mask_sample: list[bool] | None = None, + reward_original: list[float] | None = None, +) -> FinalizedGroup: + count = len(rewards) + return FinalizedGroup( + meta=KVBatchMeta( + partition_id="canonical", + task_name="train", + sample_ids=[f"group_g{i}" for i in range(count)], + fields=["input_ids"], + sequence_lengths=[3] * count, + tags=[{"weight_version": 3}] * count, + ), + group_min_wv=3, + group_max_wv=3, + staging_keys=["group_g0/call"], + reward_observations=[ + FinalizedReward(f"group_g{i}", f"group_g{i}", reward, None) + for i, reward in enumerate(rewards) + ], + reward_mask_sample=mask_sample or [False] * count, + reward_truncated=truncated or [False] * count, + reward_original=list(rewards if reward_original is None else reward_original), + valid_row_count=count, + total_row_count=count, + ) - assert buffer.size() == 0 +def _dynamic_sampling( + ctrl: Any, + *, + spares: list[dict[str, Any]], + exhausted: bool = False, + policy: str = "include", + overlong_filtering: bool = False, + lender_step: int | None = None, +) -> None: + ctrl._dynamic_sampling_enabled = True + ctrl._algo_cfg.masked_reward_policy = policy + ctrl._algo_cfg.overlong_filtering = overlong_filtering + ctrl._algo_cfg.dynamic_sampling_max_gen_batches = 10 + ctrl._filtered_prompt_budget = 10 * ctrl._algo_cfg.num_prompts_per_step + ctrl._filtered_prompts_by_step = {} + ctrl._batch_promotions = {} + ctrl._dataloader_exhausted = exhausted + ctrl._rollout_runtime.filtered_reserve = deque(spares) + ctrl._rollout_runtime.sampler_stamps_target_steps = True + ctrl._rollout_runtime.batch_shortfall = {} + ctrl._rollout_runtime.steps_allowed_short = set() + ctrl._rollout_recovery_enabled = False + ctrl._buffer.promote_ready_group = MagicMock(return_value=lender_step) + + +def _finalize_for_step_3(ctrl: Any) -> Any: + return asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=3, replacements_used=0) + ) -def test_discard_yields_to_a_group_the_train_pump_already_claimed() -> None: - """Claiming removes the slot, after which remove_group raises for it. - Mutation cuts do not exclude one another, so the train pump can claim a ready group - while the discard waits on the barrier. Treating that ValueError as a cleanup - failure would turn a benign race into a fatal RuntimeError, and clearing the rows - anyway would pull them out from under a step that is already running. - """ - ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize())) - ctrl._buffer.remove_group = AsyncMock( - side_effect=ValueError("unknown group_ids=['group']") +def test_zero_variance_group_is_rejected_before_commit_and_replaced() -> None: + """The spare comes from the filtered pool and is charged to the step's budget.""" + spare = {"idx": 7} + result = _signal_result([1.0, 1.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[spare]) + + outcome = _finalize_for_step_3(ctrl) + + assert outcome == ZeroVarianceGroup( + replacement=spare, + lineage_group_id=None, + target_step=3, + reward_observations=result.reward_observations, + zero_variance=True, ) - request = _request() - ctrl._rollout_runtime.finalizer_metrics_by_group["group"] = { - "finalize/invalid_row_rate": 0.0 - } - ctrl._rollout_runtime.finalizer_rewards_by_group["group"] = [] + ctrl._buffer.commit_finalized.assert_not_awaited() + ctrl._buffer.abort.assert_called_once_with("group") + assert not ctrl._rollout_runtime.filtered_reserve + assert ctrl._filtered_prompts_by_step == {3: 1} + assert ctrl._rollout_runtime.batch_shortfall == {} + # Not a fault-tolerance replacement, so that pool is untouched. + assert not ctrl._rollout_runtime.replacement_reserve + assert ctrl._available_finalizers.qsize() == 1 - assert asyncio.run(ctrl._discard_committed_group(request)) is False - ctrl._buffer.abort.assert_not_called() - assert ctrl._dp_client.clear_calls == [], "rows belong to the claiming step" - # The claiming step consumes the group, so its post-step release still reaches - # these; popping them here would drop metrics the trainer is about to report. - assert "group" in ctrl._rollout_runtime.finalizer_metrics_by_group - assert "group" in ctrl._rollout_runtime.finalizer_rewards_by_group +def test_zero_variance_group_borrows_a_finished_group_and_repays_the_lender() -> None: + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_signal_result([0.0, 0.0]))) + ) + _dynamic_sampling(ctrl, spares=[{"idx": 7}], lender_step=5) + ctrl._rollout_recovery_enabled = True + ctrl._rollout_recovery_ledger.__contains__.return_value = True + ctrl._rollout_manager.reserve_prompt_group.return_value = "spare-group" + + outcome = _finalize_for_step_3(ctrl) + + ctrl._buffer.promote_ready_group.assert_called_once_with(to_target_step=3) + assert ctrl._batch_promotions == {3: 1} + # The spare is owed to the lender, so it is reserved for the lender's step, + # in the same cut that discarded the rejected group's ledger entry. + assert outcome.target_step == 5 + assert outcome.lineage_group_id == "spare-group" + (discard_cut, discarded), _ = ctrl._rollout_recovery_ledger.discard_group.call_args + (reserve_cut, reserved), kwargs = ( + ctrl._rollout_manager.reserve_prompt_group.call_args + ) + assert discarded == "group" + assert reserved == {"idx": 7} + assert kwargs == {"target_step": 5} + assert reserve_cut is discard_cut + + +def test_zero_variance_group_closes_its_step_short_when_no_spare_is_left() -> None: + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_signal_result([1.0, 1.0]))) + ) + _dynamic_sampling(ctrl, spares=[], exhausted=True) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) + assert outcome.replacement is None + assert ctrl._rollout_runtime.batch_shortfall == {3: 1} + assert 3 in ctrl._rollout_runtime.steps_allowed_short + ctrl._buffer.promote_ready_group.assert_not_called() + ctrl._buffer.commit_finalized.assert_not_awaited() + + +def test_group_with_reward_spread_commits_and_leaves_the_pool_alone() -> None: + result = _signal_result([1.0, 0.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}]) + + assert _finalize_for_step_3(ctrl) is result + + ctrl._buffer.commit_finalized.assert_awaited_once() + assert len(ctrl._rollout_runtime.filtered_reserve) == 1 + assert ctrl._filtered_prompts_by_step == {} + + +def test_zero_variance_group_commits_when_dynamic_sampling_is_off() -> None: + result = _signal_result([1.0, 1.0]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + + assert _finalize_for_step_3(ctrl) is result + + ctrl._buffer.commit_finalized.assert_awaited_once() + + +@pytest.mark.parametrize( + ("policy", "rejected"), + [("exclude", True), ("include", False)], +) +def test_only_the_voting_rows_decide_the_filter(policy: str, rejected: bool) -> None: + """The truncated 0.0 is the only spread; under exclude it does not vote.""" + result = _signal_result([0.0, 1.0, 1.0, 1.0], truncated=[True, False, False, False]) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}], policy=policy, overlong_filtering=True) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) is rejected + if not rejected: + assert outcome is result + + +@pytest.mark.parametrize( + ("trained", "gym", "rejected"), + [ + # Effort shaping or a penalty made the trained rewards differ; the Gym + # reward did not, so the group carries no task signal. + ([1.9, 1.0], [1.0, 1.0], True), + ([0.0, 1.0], [1.0, 1.0], True), + # Only the trained rewards are uniform; the Gym rewards still differ. + ([0.0, 0.0], [0.0, 1.0], False), + ], +) +def test_the_filter_reads_the_original_reward_not_the_trained_reward( + trained: list[float], gym: list[float], rejected: bool +) -> None: + result = _signal_result(trained, reward_original=gym) + ctrl = _controller(SimpleNamespace(finalize=_RemoteFinalize(result=result))) + _dynamic_sampling(ctrl, spares=[{"idx": 7}]) + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, ZeroVarianceGroup) is rejected + + +def test_below_valid_fraction_is_decided_before_zero_variance() -> None: + """A group that fails both takes a fault-tolerance spare, not a filtered one.""" + ctrl = _controller( + SimpleNamespace(finalize=_RemoteFinalize(result=_below_threshold_result())) + ) + _below_threshold(ctrl, spares=[{"idx": 1}]) + _dynamic_sampling(ctrl, spares=[{"idx": 2}]) + ctrl._rollout_runtime.replacement_reserve = deque([{"idx": 1}]) + ctrl._rollout_runtime.sampler_stamps_target_steps = True + + outcome = _finalize_for_step_3(ctrl) + + assert isinstance(outcome, BelowValidFractionGroup) + assert outcome.replacement == {"idx": 1} + assert len(ctrl._rollout_runtime.filtered_reserve) == 1 + assert ctrl._filtered_prompts_by_step == {} def test_actor_rpc_failure_is_fatal_and_does_not_retry_or_requeue_actor() -> None: @@ -509,7 +848,11 @@ def test_actor_rpc_failure_is_fatal_and_does_not_retry_or_requeue_actor() -> Non request = _request() with pytest.raises(RuntimeError, match="actor died after submission"): - asyncio.run(ctrl._finalize_with_actor(request, {})) + asyncio.run( + ctrl._finalize_with_actor( + request, {}, target_step=None, replacements_used=0 + ) + ) assert finalize.calls == [request] assert ctrl._available_finalizers.empty() @@ -532,7 +875,11 @@ def test_missing_actor_metadata_cleans_known_canonical_and_staging_ownership() - ctrl = _controller(actor) with pytest.raises(RuntimeError, match="no metadata for non-dropped group"): - asyncio.run(ctrl._finalize_with_actor(_request(), {})) + asyncio.run( + ctrl._finalize_with_actor( + _request(), {}, target_step=None, replacements_used=0 + ) + ) assert ctrl._dp_client.clear_calls == [ {"sample_ids": ["group_g0"], "partition_id": "canonical"}, @@ -562,7 +909,9 @@ def test_dropped_actor_group_cleans_ownership_and_returns_uncommitted() -> None: actor = SimpleNamespace(finalize=_RemoteFinalize(result=result)) ctrl = _controller(actor) - committed = asyncio.run(ctrl._finalize_with_actor(_request(), {})) + committed = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=None, replacements_used=0) + ) assert committed is None assert ctrl._dp_client.clear_calls == [ diff --git a/tests/unit/single_controller/test_ppo_setup.py b/tests/unit/single_controller/test_ppo_setup.py index dd496f18025..770292fafa9 100644 --- a/tests/unit/single_controller/test_ppo_setup.py +++ b/tests/unit/single_controller/test_ppo_setup.py @@ -841,14 +841,11 @@ def test_accepts_shrink_because_filtering_replaces_regardless(self): _dynamic_sampling_config(on_dropped_prompt="shrink") ) - def test_rejects_token_capture(self): - # The capture path cannot borrow a finished group from a later step, so every - # drop would stall its step for a full fresh rollout. + def test_accepts_token_capture(self): mc = _dynamic_sampling_config() mc.token_capture.enabled = True - with pytest.raises(ValueError, match="token_capture.enabled"): - validate_single_controller_config(mc) + validate_single_controller_config(mc) def test_rejects_a_zero_generation_batch_allowance(self): mc = _dynamic_sampling_config(max_gen_batches=0) diff --git a/tests/unit/single_controller/test_rollout_pump.py b/tests/unit/single_controller/test_rollout_pump.py index fa3f2bc7b53..090b0cba8f6 100644 --- a/tests/unit/single_controller/test_rollout_pump.py +++ b/tests/unit/single_controller/test_rollout_pump.py @@ -53,6 +53,10 @@ RolloutRecoveryConfig, ) from nemo_rl.algorithms.single_controller_utils.setup import SingleControllerActorArgs +from nemo_rl.algorithms.single_controller_utils.utils import ( + BelowValidFractionGroup, + ZeroVarianceGroup, +) from nemo_rl.data.interfaces import DatumSpec from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.experience.rollout_manager import RolloutManager, RolloutOutcome @@ -83,6 +87,11 @@ from tests.unit.single_controller._rollout_checkpoints import ( attach_rollout_checkpointing, ) +from tests.unit.single_controller.test_finalizer_lifecycle import ( + _DataPlaneClient, + _request, + _signal_result, +) def _failure_cfg( @@ -121,6 +130,8 @@ def _init_pump_ledgers(ctrl: Any) -> None: # Dynamic sampling off: the filter never fires, so no test here draws on its # budget unless it opts in. ctrl._dynamic_sampling_enabled = False + ctrl._is_ppo = False + ctrl._filtered_spares_claimed = 0 ctrl._filtered_prompt_budget = 0 ctrl._rollout_runtime.filtered_reserve = deque() ctrl._filtered_prompts_by_step = {} @@ -1476,7 +1487,6 @@ def test_min_valid_fraction_counts_rollouts_independently_of_segments( row_counts: tuple[int, int], threshold: float | None, forest_training: bool, - capsys: pytest.CaptureFixture[str], ) -> None: valid_rows, total_rows = row_counts # Three of four owners verified, regardless of how many rows they produced. @@ -1497,8 +1507,36 @@ def test_min_valid_fraction_counts_rollouts_independently_of_segments( else {} ), ) - fraction = 0.75 if forest_training else valid_rows / total_rows - dropped = threshold is not None and fraction < threshold + counts = (3, 4) if forest_training else row_counts + dropped = threshold is not None and counts[0] / counts[1] < threshold + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._master_config = SimpleNamespace( + token_capture=SimpleNamespace( + min_valid_fraction_per_group=threshold, forest_training=forest_training + ), + ) + + assert ctrl._valid_fraction_counts(finalized) == (counts if dropped else None) + + +@pytest.mark.parametrize("has_spare", [False, True]) +def test_below_valid_fraction_outcome_substitutes_or_closes_short( + has_spare: bool, capsys: pytest.CaptureFixture[str] +) -> None: + """The pump acts on the finalize cut's decision instead of re-deciding it.""" + spare = {"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1} + rejected = BelowValidFractionGroup( + replacement=spare if has_spare else None, + lineage_group_id="spare-group" if has_spare else None, + target_step=0, + reward_observations=[], + zero_variance=True, + valid_count=3, + total_count=4, + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) async def run() -> None: ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) @@ -1508,7 +1546,7 @@ async def run() -> None: ctrl._master_config = SimpleNamespace( grpo=GRPOConfig.model_construct(max_num_epochs=1), token_capture=SimpleNamespace( - min_valid_fraction_per_group=threshold, forest_training=forest_training + min_valid_fraction_per_group=0.8, forest_training=False ), ) ctrl._algo_cfg = ctrl._master_config.grpo @@ -1523,9 +1561,11 @@ async def run() -> None: ) _init_pump_ledgers(ctrl) ctrl._finalizer_actors = [object()] - ctrl._finalize_with_actor = AsyncMock(return_value=finalized) - ctrl._discard_committed_group = AsyncMock() - ctrl._sampler = WindowedSampler(None, max_staleness_versions=1) + ctrl._finalize_with_actor = AsyncMock(side_effect=[rejected, committed]) + # in_order stamps the batch for step 0, so the step the finalize cut + # replaces into (or credits short) is visible. + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) ctrl._dataloader = [ BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) ] @@ -1541,20 +1581,405 @@ async def run() -> None: await ctrl._rollout_pump() - assert ctrl._rollout_manager.stats.committed == int(not dropped) - assert ctrl._buffer_capacity._value == int(dropped) - assert ctrl._inflight_rollouts == 0 + # The finalize cut decides replacement and shortfall from these, so the + # pump must pass the stamped step and its replacement count. + assert [call.kwargs for call in ctrl._finalize_with_actor.await_args_list] == [ + {"target_step": 0, "replacements_used": 0} + ] + ([{"target_step": 0, "replacements_used": 1}] if has_spare else []) + # The finalize cut owns the shortfall credit; the pump adds none. + assert ctrl._rollout_runtime.batch_shortfall == {} assert ctrl._rollout_exhausted.is_set() - if dropped: - ctrl._discard_committed_group.assert_awaited_once_with(request) + assert ctrl._rollout_manager.stats.committed == int(has_spare) + assert ctrl._buffer_capacity._value == int(not has_spare) + assert ctrl._inflight_rollouts == 0 + generate = ctrl._rollout_manager.generate_for_finalization + if has_spare: + # The spare runs under the lineage reserved inside the finalize cut. + assert generate.await_count == 2 + assert generate.await_args.args == (spare,) + assert generate.await_args.kwargs["lineage_group_id"] == "spare-group" + ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_not_called() + else: + assert generate.await_count == 1 ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_called_once() + + asyncio.run(run()) + assert "(3/4 < 0.8)" in capsys.readouterr().out + + +@pytest.mark.parametrize("has_spare", [False, True]) +def test_zero_variance_outcome_substitutes_from_the_filtered_pool_or_closes_short( + has_spare: bool, capsys: pytest.CaptureFixture[str] +) -> None: + """A filtered group draws only the dynamic-sampling budget, never a failure retry.""" + spare = {"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1} + # Step 1 lent the finished group that filled step 0, so the spare repays step 1. + rejected = ZeroVarianceGroup( + replacement=spare if has_spare else None, + lineage_group_id="spare-group" if has_spare else None, + target_step=1, + reward_observations=[], + zero_variance=True, + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=None, forest_training=False + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + request = SimpleNamespace(group_id="group") + ctrl._rollout_manager = SimpleNamespace( + generate_for_finalization=AsyncMock(return_value=(request, {})), + record_finalizer_dropped_prompt=MagicMock(), + record_group_rewards=MagicMock(), + stats=SimpleNamespace(committed=0, zero_variance_groups=0), + ) + _init_pump_ledgers(ctrl) + ctrl._finalizer_actors = [object()] + ctrl._finalize_with_actor = AsyncMock(side_effect=[rejected, committed]) + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + await ctrl._rollout_pump() + + # The spare is generated for the step the finalize cut reserved it for, and + # it does not spend the fault-tolerance budget. + assert [call.kwargs for call in ctrl._finalize_with_actor.await_args_list] == [ + {"target_step": 0, "replacements_used": 0} + ] + ([{"target_step": 1, "replacements_used": 0}] if has_spare else []) + assert ctrl._rollout_manager.stats.zero_variance_groups == 1 + assert ctrl._rollout_manager.stats.committed == int(has_spare) + assert ctrl._buffer_capacity._value == int(not has_spare) + assert ctrl._inflight_rollouts == 0 + # A filtered group is a healthy rollout, not an infrastructure drop. + ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_not_called() + generate = ctrl._rollout_manager.generate_for_finalization + if has_spare: + assert generate.await_count == 2 + assert generate.await_args.args == (spare,) + assert generate.await_args.kwargs["lineage_group_id"] == "spare-group" + # Counted against the step that was repaid. + assert ctrl._batch_replacements == {1: 1} + tallies = ctrl._rollout_manager.record_group_rewards.call_args_list + assert [call.kwargs["replacement_index"] for call in tallies] == [0, 1] else: - ctrl._discard_committed_group.assert_not_awaited() - ctrl._rollout_manager.record_finalizer_dropped_prompt.assert_not_called() + assert generate.await_count == 1 + + asyncio.run(run()) + assert "its voting rewards carry no signal" in capsys.readouterr().out + + +def test_a_group_rejected_for_both_reasons_tallies_each_attempt_once( + capsys: pytest.CaptureFixture[str], +) -> None: + """Fault-tolerance and filtered replacements share one tally index sequence.""" + below_valid = BelowValidFractionGroup( + replacement={"message_log": [[{"role": "user", "content": "a"}]], "idx": 1}, + lineage_group_id=None, + target_step=0, + reward_observations=[], + zero_variance=False, + valid_count=3, + total_count=4, + ) + zero_variance = ZeroVarianceGroup( + replacement={"message_log": [[{"role": "user", "content": "b"}]], "idx": 2}, + lineage_group_id=None, + target_step=0, + reward_observations=[], + zero_variance=True, + ) + committed = FinalizedGroup( + meta=None, group_min_wv=0, group_max_wv=0, staging_keys=[] + ) + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=0.8, forest_training=False + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + request = SimpleNamespace(group_id="group") + ctrl._rollout_manager = SimpleNamespace( + generate_for_finalization=AsyncMock(return_value=(request, {})), + record_finalizer_dropped_prompt=MagicMock(), + record_group_rewards=MagicMock(), + stats=SimpleNamespace(committed=0, zero_variance_groups=0), + ) + _init_pump_ledgers(ctrl) + ctrl._finalizer_actors = [object()] + ctrl._finalize_with_actor = AsyncMock( + side_effect=[below_valid, zero_variance, committed] + ) + ctrl._buffer = _RecordingBuffer() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + await ctrl._rollout_pump() + + # Only the fault-tolerance replacement spends that budget. + assert [ + call.kwargs["replacements_used"] + for call in ctrl._finalize_with_actor.await_args_list + ] == [0, 1, 1] + tallies = ctrl._rollout_manager.record_group_rewards.call_args_list + assert [call.kwargs["replacement_index"] for call in tallies] == [0, 1, 2] + assert ctrl._rollout_manager.stats.zero_variance_groups == 1 + assert ctrl._rollout_manager.stats.committed == 1 + + asyncio.run(run()) + capsys.readouterr() + + +def _claiming_controller(*, spares: int, exhausted: bool = False) -> Any: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._dynamic_sampling_enabled = True + ctrl._rollout_runtime = RolloutRuntimeState() + ctrl._rollout_runtime.filtered_reserve = deque({"idx": i} for i in range(spares)) + ctrl._filtered_spares_claimed = 0 + ctrl._dataloader_exhausted = exhausted + return ctrl + + +def test_claiming_a_filtered_spare_waits_until_one_is_unclaimed() -> None: + """Two finalizing groups cannot both count on the pool's only spare.""" + + async def run() -> None: + ctrl = _claiming_controller(spares=1) + + async def second_claim() -> int: + async with ctrl._claim_filtered_spare(): + return ctrl._filtered_spares_claimed + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 1 + waiting = asyncio.create_task(second_claim()) + await asyncio.sleep(0.25) + assert not waiting.done() + # The pump refilled the pool, so a second spare can be promised. + ctrl._rollout_runtime.filtered_reserve.append({"idx": 9}) + assert await asyncio.wait_for(waiting, timeout=5) == 2 + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_a_claim_is_released_when_finalization_fails() -> None: + async def run() -> None: + ctrl = _claiming_controller(spares=1) + + with pytest.raises(RuntimeError, match="boom"): + async with ctrl._claim_filtered_spare(): + raise RuntimeError("boom") + + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_no_claim_is_made_once_the_dataloader_is_exhausted() -> None: + """Nothing will refill an empty pool, so waiting for it would hang the group.""" + + async def run() -> None: + ctrl = _claiming_controller(spares=0, exhausted=True) + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_claiming_does_nothing_without_dynamic_sampling() -> None: + async def run() -> None: + ctrl = _claiming_controller(spares=0) + ctrl._dynamic_sampling_enabled = False + + async with ctrl._claim_filtered_spare(): + assert ctrl._filtered_spares_claimed == 0 + + asyncio.run(run()) + + +def test_pump_claims_and_pops_a_filtered_spare_for_a_real_zero_variance_rejection() -> ( + None +): + """The pump drives a real finalize that classifies and pops the filtered spare. + + This covers the ``_claim_filtered_spare`` wrapper around the real + ``_finalize_with_actor``: a finalized group whose voting rewards carry no signal + is classified (not mocked) inside the finalize cut, rejected, its spare popped + from the filtered pool, and the step refilled by the committed replacement. + """ + # The first group is uniform (zero variance -> rejected); its filtered spare is + # the second group, which has reward spread and commits. + results = iter([_signal_result([1.0, 1.0]), _signal_result([1.0, 0.0])]) + + class _ClassifyingRolloutManager: + """Owns the voting settings and classifies finalized capture groups.""" + + def __init__(self) -> None: + self._masked_reward_policy = "include" + self._overlong_filtering = False + self.stats = SimpleNamespace(committed=0, zero_variance_groups=0) + self.record_group_rewards = MagicMock() + self.record_finalizer_dropped_prompt = MagicMock() + self.record_canonical_publication = MagicMock() + self.generated: list[Any] = [] + + def _is_zero_variance_finalized(self, finalized: FinalizedGroup) -> bool: + return RolloutManager._is_zero_variance_finalized(self, finalized) + + async def generate_for_finalization( + self, prompt: Any, *, target_step: int | None = None, **kwargs: Any + ) -> Any: + del target_step, kwargs + self.generated.append(prompt) + return _request(), {} + + manager = _ClassifyingRolloutManager() + + class _ResultActor: + """Finalizer actor returning the next pre-built result each call.""" + + def __init__(self) -> None: + self.finalize = SimpleNamespace(remote=self._remote) + + async def _remote(self, request: Any) -> FinalizedGroup: + del request + return next(results) + + actor = _ResultActor() + + async def run() -> None: + ctrl = object.__new__(SingleControllerActor.__ray_metadata__.modified_class) + ctrl._async_cfg = SimpleNamespace( + max_inflight_prompts=1, diagnostics=False, rollout_failure=_failure_cfg() + ) + ctrl._master_config = SimpleNamespace( + grpo=GRPOConfig.model_construct(max_num_epochs=1, num_prompts_per_step=1), + token_capture=SimpleNamespace( + min_valid_fraction_per_group=None, + forest_training=False, + staging_partition="staging", + ), + ) + ctrl._algo_cfg = ctrl._master_config.grpo + ctrl._is_ppo = False + ctrl._partition_id = "canonical" + ctrl._dp_client = _DataPlaneClient() + ctrl._rollout_manager = manager + _init_pump_ledgers(ctrl) + + # Run real controller finalization with fake actor and data-plane endpoints. + ctrl._data_plane_checkpoint_barrier = DataPlaneCheckpointBarrier() + ctrl._available_finalizers = asyncio.Queue() + ctrl._available_finalizers.put_nowait(actor) + ctrl._active_finalizers = 0 + ctrl._finalizer_waiters = 0 + ctrl._finalizer_unknown_outcomes = 0 + ctrl._rollout_recovery_ledger = MagicMock() + ctrl._rollout_recovery_ledger.__contains__.return_value = False + ctrl._rollout_recovery_enabled = False + ctrl._refit_cuts = SimpleNamespace(tokens_by_version=lambda _: {}) + + # Dynamic sampling on, with one spare in the filtered pool for the + # zero-variance rejection to consume. + ctrl._dynamic_sampling_enabled = True + ctrl._dataloader_exhausted = False + ctrl._filtered_spares_claimed = 0 + ctrl._filtered_prompt_budget = 10 + ctrl._filtered_prompts_by_step = {} + ctrl._rollout_runtime.filtered_reserve = deque( + [{"message_log": [[{"role": "user", "content": "spare"}]], "idx": 1}] + ) + ctrl._rollout_runtime.steps_allowed_short = set() + + ctrl._buffer = _RecordingBuffer() + ctrl._buffer.commit_finalized = AsyncMock() + ctrl._buffer.abort = MagicMock() + ctrl._sampler = InOrderSampler(ctrl._buffer, max_lookahead_versions=1) + ctrl._finalizer_actors = [actor] + ctrl._dataloader = [ + BatchedDataDict({"message_log": [[{"role": "user", "content": "prompt"}]]}) + ] + ctrl._rollout_permitted = asyncio.Event() + ctrl._rollout_permitted.set() + ctrl._rollout_exhausted = asyncio.Event() + ctrl._buffer_capacity = asyncio.Semaphore(1) + ctrl._inflight_rollouts = 0 + ctrl._inflight_by_group_id = {} + ctrl._dispatched_rollouts = set() + ctrl._trainer_version = 0 + ctrl._current_epoch = 0 + + # The pump must claim the spare before the finalize cut pops it, so record + # the claim count held inside finalization. + claims_during_finalize: list[int] = [] + finalize = ctrl._finalize_with_actor + + async def finalize_with_claim(*args: Any, **kwargs: Any) -> Any: + claims_during_finalize.append(ctrl._filtered_spares_claimed) + return await finalize(*args, **kwargs) + + ctrl._finalize_with_actor = finalize_with_claim + + await ctrl._rollout_pump() + + # The zero-variance group was classified and rejected, its spare popped + # from the filtered pool and charged, and the replacement committed. + assert manager.stats.zero_variance_groups == 1 + assert manager.stats.committed == 1 + assert not ctrl._rollout_runtime.filtered_reserve + assert ctrl._filtered_prompts_by_step == {0: 1} + # The pump held the spare's claim across finalization, then released it. + assert claims_during_finalize[0] == 1 + assert ctrl._filtered_spares_claimed == 0 + # Two generations: the rejected group and its filtered replacement. + assert len(manager.generated) == 2 + # Both tallies recorded; the rejection carried the zero-variance verdict. + tallies = manager.record_group_rewards.call_args_list + assert [call.kwargs["zero_variance"] for call in tallies] == [True, False] asyncio.run(run()) - if dropped and forest_training: - assert "(3/4 < 0.8)" in capsys.readouterr().out def test_actor_path_releases_generation_permit_before_finalization() -> None: @@ -1590,15 +2015,19 @@ async def _delayed_finalize( rollout_metrics: dict[str, Any], *, target_step: int | None = None, + replacements_used: int = 0, ) -> bool: nonlocal finalizers_started del request, rollout_metrics, target_step finalizers_started += 1 await release_finalizers.wait() - return SimpleNamespace( + return FinalizedGroup( + meta=None, + group_min_wv=0, + group_max_wv=0, + staging_keys=[], valid_row_count=1, total_row_count=1, - reward_observations=[], ) controller_cls = SingleControllerActor.__ray_metadata__.modified_class @@ -1762,15 +2191,19 @@ async def _main() -> None: async def _finalize( request: Any, rollout_metrics: dict[str, Any], + **_: Any, ) -> Any: async with ctrl._data_plane_checkpoint_barrier.mutation() as cut: manager.recovery_ledger.discard_group(cut, request.group_id) if not committed: return None - return SimpleNamespace( + return FinalizedGroup( + meta=None, + group_min_wv=0, + group_max_wv=0, + staging_keys=[], valid_row_count=1, total_row_count=1, - reward_observations=[], ) ctrl._finalize_with_actor = _finalize diff --git a/tests/unit/single_controller/test_rollout_trace.py b/tests/unit/single_controller/test_rollout_trace.py index 9046e886499..8fcee59138d 100644 --- a/tests/unit/single_controller/test_rollout_trace.py +++ b/tests/unit/single_controller/test_rollout_trace.py @@ -482,7 +482,10 @@ def test_a_published_group_traces_each_call_and_its_version(tmp_path: Path) -> N # The v4 refit paused c1 after 6 of its 10 tokens. ctrl._refit_cuts.record(3, 4, [["c1", 6]]) - assert asyncio.run(ctrl._finalize_with_actor(_request(), {})) is result + finalized = asyncio.run( + ctrl._finalize_with_actor(_request(), {}, target_step=None, replacements_used=0) + ) + assert finalized is result ctrl._rollout_trace.close() (published,) = [ diff --git a/tests/unit/single_controller/test_step_reward_metrics.py b/tests/unit/single_controller/test_step_reward_metrics.py index d3d7337abcb..3750741aef1 100644 --- a/tests/unit/single_controller/test_step_reward_metrics.py +++ b/tests/unit/single_controller/test_step_reward_metrics.py @@ -19,7 +19,12 @@ import pytest from nemo_rl.data_plane.schema import MASK_SAMPLE -from nemo_rl.experience.dynamic_sampling import rewards_carry_no_signal +from nemo_rl.experience.dynamic_sampling import ( + rewards_carry_no_signal, + voting_rewards, + voting_rewards_from_finalized, + voting_rewards_from_record, +) from nemo_rl.experience.rollout_manager import RolloutManager @@ -49,9 +54,10 @@ def test_rewards_carry_no_signal(rewards, expected): assert rewards_carry_no_signal(rewards) is expected -def _completion(reward, *, truncated=False, mask_sample=False): +def _completion(reward, *, truncated=False, mask_sample=False, reward_original=None): return SimpleNamespace( reward=reward, + reward_original=reward_original, truncated=truncated, env_extras={MASK_SAMPLE: True} if mask_sample else None, ) @@ -63,9 +69,9 @@ def _record(completions, *, loss_multiplier=1.0): def test_zero_variance_group_predicate_reads_completion_rewards(): record = _record([_completion(1.0), _completion(1.0)]) - assert _manager()._is_zero_variance_group(record) is True + assert rewards_carry_no_signal(_manager()._voting_rewards(record)) is True record.completions.append(_completion(0.0)) - assert _manager()._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(_manager()._voting_rewards(record)) is False def test_only_the_voting_rows_decide_whether_a_group_is_trivial(): @@ -82,15 +88,15 @@ def test_only_the_voting_rows_decide_whether_a_group_is_trivial(): excluding = _manager("exclude", overlong_filtering=True) including = _manager("include", overlong_filtering=True) - assert excluding._is_zero_variance_group(record) is True - assert including._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(excluding._voting_rewards(record)) is True + assert rewards_carry_no_signal(including._voting_rewards(record)) is False def test_a_flagged_row_only_votes_under_include(): record = _record([_completion(0.0, mask_sample=True), _completion(1.0)]) - assert _manager("exclude")._is_zero_variance_group(record) is True - assert _manager("include")._is_zero_variance_group(record) is False + assert rewards_carry_no_signal(_manager("exclude")._voting_rewards(record)) is True + assert rewards_carry_no_signal(_manager("include")._voting_rewards(record)) is False def test_a_zero_weight_group_is_trivial_under_either_policy(): @@ -99,7 +105,71 @@ def test_a_zero_weight_group_is_trivial_under_either_policy(): record = _record([_completion(0.0), _completion(1.0)], loss_multiplier=0.0) for policy in ("include", "exclude", None): - assert _manager(policy)._is_zero_variance_group(record) is True + assert rewards_carry_no_signal(_manager(policy)._voting_rewards(record)) is True + + +@pytest.mark.parametrize( + ("policy", "overlong_filtering", "expected"), + [ + ("include", False, [0.0, 1.0, 0.5, 0.25]), + ("include", True, [0.0, 1.0, 0.5, 0.25]), + ("exclude", False, [0.0, 0.5, 0.25]), + ("exclude", True, [0.5, 0.25]), + (None, True, [0.0, 1.0, 0.5, 0.25]), + ], +) +def test_the_record_and_finalized_adapters_pick_the_same_voters( + policy, overlong_filtering, expected +): + """The capture path has a finalized group, not a record; both must pick the same voters.""" + completions = [ + _completion(0.0, truncated=True), + _completion(1.0, mask_sample=True), + _completion(0.5), + _completion(0.25), + ] + + from_flags = voting_rewards( + [c.reward for c in completions], + mask_sample=[c.env_extras is not None for c in completions], + truncated=[c.truncated for c in completions], + loss_multiplier=1.0, + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + + assert from_flags == expected + assert from_flags == voting_rewards_from_record( + _record(completions), + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + finalized = SimpleNamespace( + reward_original=[c.reward for c in completions], + reward_mask_sample=[c.env_extras is not None for c in completions], + reward_truncated=[c.truncated for c in completions], + loss_multiplier=1.0, + ) + assert from_flags == voting_rewards_from_finalized( + finalized, + masked_reward_policy=policy, + overlong_filtering=overlong_filtering, + ) + + +def test_voting_rewards_read_the_original_reward_when_a_completion_has_one(): + """Effort shaping and penalties rewrite ``reward``; voting reads the Gym value.""" + completions = [ + _completion(1.9, reward_original=1.0), + _completion(0.0, reward_original=1.0), + _completion(0.5), + ] + + assert voting_rewards_from_record( + _record(completions), + masked_reward_policy="include", + overlong_filtering=False, + ) == [1.0, 1.0, 0.5] def test_unfiltered_reward_counts_kept_and_dropped_groups(): diff --git a/tests/unit/test_effort_shaping.py b/tests/unit/test_effort_shaping.py index b751c1001f5..607438da719 100644 --- a/tests/unit/test_effort_shaping.py +++ b/tests/unit/test_effort_shaping.py @@ -355,6 +355,8 @@ def test_run_rollouts_shapes_completion_reward_and_emits_low_metrics(): # length_reward = min(1, 1.0 * (1 - 100/1000)) = 0.9 -> 1.0 + 1.0 * 0.9 = 1.9 assert completions[0].reward == pytest.approx(1.9) + # Dynamic sampling filters on the reward from before shaping. + assert completions[0].reward_original == pytest.approx(1.0) # Downstream reward metrics are computed from the shaped Completion. assert metrics["total_reward/mean"] == pytest.approx(1.9) assert metrics["mean_length_reward_low"] == pytest.approx(0.9)